{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41018985-0127-4157-b694-8805685500ad",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-27T15:08:15.581115Z",
     "iopub.status.busy": "2024-08-27T15:08:15.580355Z",
     "iopub.status.idle": "2024-08-27T15:08:15.588687Z",
     "shell.execute_reply": "2024-08-27T15:08:15.588337Z",
     "shell.execute_reply.started": "2024-08-27T15:08:15.581022Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
      "env: LLM_API_KEY=<替换为自己的key>\n"
     ]
    }
   ],
   "source": [
    "%env LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1\n",
    "%env LLM_API_KEY=<替换为自己的key>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d48b549b-48b5-4e8c-bdb6-458ebb393f4f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:31:14.379962Z",
     "iopub.status.busy": "2024-08-15T15:31:14.379744Z",
     "iopub.status.idle": "2024-08-15T15:31:14.382426Z",
     "shell.execute_reply": "2024-08-15T15:31:14.381983Z",
     "shell.execute_reply.started": "2024-08-15T15:31:14.379944Z"
    }
   },
   "outputs": [],
   "source": [
    "%%capture --no-stderr\n",
    "!pip install -U langchain langchain_community langchain_openai pypdf sentence_transformers chromadb shutil openpyxl rank_bm25 jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "df8dd3ad-e795-4479-8c8c-900191f01c86",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:31:15.674614Z",
     "iopub.status.busy": "2024-08-15T15:31:15.673838Z",
     "iopub.status.idle": "2024-08-15T15:31:15.685895Z",
     "shell.execute_reply": "2024-08-15T15:31:15.683708Z",
     "shell.execute_reply.started": "2024-08-15T15:31:15.674545Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "langchain                     0.2.10\n",
      "langchain_core                0.2.28\n",
      "langchain_community           0.2.9\n",
      "pypdf                         4.3.1\n",
      "sentence_transformers         3.0.1\n",
      "chromadb                      0.5.4\n"
     ]
    }
   ],
   "source": [
    "import langchain, langchain_community, pypdf, sentence_transformers, chromadb, langchain_core\n",
    "\n",
    "for module in (langchain, langchain_core, langchain_community, pypdf, sentence_transformers, chromadb):\n",
    "    print(f\"{module.__name__:<30}{module.__version__}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "1e2c72b8-ee12-4130-af88-699998aa230c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:31:17.727493Z",
     "iopub.status.busy": "2024-08-15T15:31:17.727063Z",
     "iopub.status.idle": "2024-08-15T15:31:17.731504Z",
     "shell.execute_reply": "2024-08-15T15:31:17.730488Z",
     "shell.execute_reply.started": "2024-08-15T15:31:17.727462Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "841d2b02-ad06-40d2-b11f-c7adccec6ca2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:32:20.561262Z",
     "iopub.status.busy": "2024-08-15T15:32:20.560462Z",
     "iopub.status.idle": "2024-08-15T15:32:20.570632Z",
     "shell.execute_reply": "2024-08-15T15:32:20.568901Z",
     "shell.execute_reply.started": "2024-08-15T15:32:20.561189Z"
    }
   },
   "outputs": [],
   "source": [
    "expr_version = 'retrieval_v4_bm25_hybrid'\n",
    "\n",
    "preprocess_output_dir = os.path.join(os.path.pardir, 'outputs', 'v1_20240713')\n",
    "expr_dir = os.path.join(os.path.pardir, 'experiments', expr_version)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7e81e3-4c82-4842-aef5-7592caaf1d39",
   "metadata": {},
   "source": [
    "# 读取文档"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "da15f02e-3131-43fb-81c5-f4da615c449b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:32:21.761390Z",
     "iopub.status.busy": "2024-08-15T15:32:21.760601Z",
     "iopub.status.idle": "2024-08-15T15:32:21.770836Z",
     "shell.execute_reply": "2024-08-15T15:32:21.768477Z",
     "shell.execute_reply.started": "2024-08-15T15:32:21.761321Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import PyPDFLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e6920e29-bc7d-4635-be06-d151eaf0e100",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:32:21.927907Z",
     "iopub.status.busy": "2024-08-15T15:32:21.927083Z",
     "iopub.status.idle": "2024-08-15T15:32:23.139326Z",
     "shell.execute_reply": "2024-08-15T15:32:23.138887Z",
     "shell.execute_reply.started": "2024-08-15T15:32:21.927835Z"
    }
   },
   "outputs": [],
   "source": [
    "loader = PyPDFLoader(os.path.join(os.path.pardir, 'data', '2024全球经济金融展望报告.pdf'))\n",
    "documents = loader.load()\n",
    "\n",
    "qa_df = pd.read_excel(os.path.join(preprocess_output_dir, 'question_answer.xlsx'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "841ec659-4ad7-4e1f-b1ea-3477bf97fde3",
   "metadata": {},
   "source": [
    "# 文档切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "74fe856a-7c19-4c3c-bb30-7abfa6298f74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:32:23.140140Z",
     "iopub.status.busy": "2024-08-15T15:32:23.140007Z",
     "iopub.status.idle": "2024-08-15T15:32:23.143428Z",
     "shell.execute_reply": "2024-08-15T15:32:23.143098Z",
     "shell.execute_reply.started": "2024-08-15T15:32:23.140127Z"
    }
   },
   "outputs": [],
   "source": [
    "from uuid import uuid4\n",
    "import os\n",
    "import pickle\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "def split_docs(documents, filepath, chunk_size=400, chunk_overlap=40, seperators=['\\n\\n\\n', '\\n\\n'], force_split=False):\n",
    "    if os.path.exists(filepath) and not force_split:\n",
    "        print('found cache, restoring...')\n",
    "        return pickle.load(open(filepath, 'rb'))\n",
    "\n",
    "    splitter = RecursiveCharacterTextSplitter(\n",
    "        chunk_size=chunk_size,\n",
    "        chunk_overlap=chunk_overlap,\n",
    "        separators=seperators\n",
    "    )\n",
    "    split_docs = splitter.split_documents(documents)\n",
    "    for chunk in split_docs:\n",
    "        chunk.metadata['uuid'] = str(uuid4())\n",
    "\n",
    "    pickle.dump(split_docs, open(filepath, 'wb'))\n",
    "\n",
    "    return split_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "aa25540d-0504-4ae7-9804-9e3862b132d5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:32:23.143956Z",
     "iopub.status.busy": "2024-08-15T15:32:23.143831Z",
     "iopub.status.idle": "2024-08-15T15:32:23.158032Z",
     "shell.execute_reply": "2024-08-15T15:32:23.157552Z",
     "shell.execute_reply.started": "2024-08-15T15:32:23.143943Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found cache, restoring...\n"
     ]
    }
   ],
   "source": [
    "splitted_docs = split_docs(documents, os.path.join(preprocess_output_dir, 'split_docs.pkl'), chunk_size=500, chunk_overlap=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "220dbc3a-fceb-4e49-a3f1-01e16660b2a6",
   "metadata": {},
   "source": [
    "# 检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "8598a11c-25d8-4af1-a98b-06a8c394e261",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:32:24.561842Z",
     "iopub.status.busy": "2024-08-15T15:32:24.561610Z",
     "iopub.status.idle": "2024-08-15T15:32:24.566404Z",
     "shell.execute_reply": "2024-08-15T15:32:24.565819Z",
     "shell.execute_reply.started": "2024-08-15T15:32:24.561824Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device: cuda\n"
     ]
    }
   ],
   "source": [
    "from langchain.embeddings import HuggingFaceBgeEmbeddings\n",
    "import torch\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(f'device: {device}')\n",
    "\n",
    "def get_embeddings(model_path):\n",
    "    embeddings = HuggingFaceBgeEmbeddings(\n",
    "        model_name=model_path,\n",
    "        model_kwargs={'device': device},\n",
    "        encode_kwargs={'normalize_embeddings': True},\n",
    "        # show_progress=True\n",
    "        query_instruction='为这个句子生成表示以用于检索相关文章：'\n",
    "    )\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "663ef1a4-5866-4f6b-8d9d-4724f62142cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:32:25.006309Z",
     "iopub.status.busy": "2024-08-15T15:32:25.006132Z",
     "iopub.status.idle": "2024-08-15T15:32:39.013580Z",
     "shell.execute_reply": "2024-08-15T15:32:39.013099Z",
     "shell.execute_reply.started": "2024-08-15T15:32:25.006295Z"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "# from langchain_community.vectorstores import Qdrant\n",
    "from langchain_community.vectorstores import Chroma\n",
    "from langchain.retrievers import BM25Retriever, EnsembleRetriever\n",
    "import jieba\n",
    "import shutil\n",
    "\n",
    "# 微调后的模型\n",
    "EMBEDDING_MODEL_PATH = 'stevenluo/bge-large-zh-v1.5-ft-v4'\n",
    "\n",
    "# 确保结果可复现性\n",
    "persist_directory = os.path.join(expr_dir, 'chroma', 'bge')\n",
    "shutil.rmtree(persist_directory, ignore_errors=True)\n",
    "\n",
    "embeddings = get_embeddings(EMBEDDING_MODEL_PATH)\n",
    "vector_db = Chroma.from_documents(\n",
    "    splitted_docs,\n",
    "    embedding=embeddings,\n",
    "    persist_directory=persist_directory\n",
    ")\n",
    "vanilla_bm25_retriever = BM25Retriever.from_documents(splitted_docs)\n",
    "chz_cut_bm25_retriever = BM25Retriever.from_documents(splitted_docs, preprocess_func=lambda text: list(jieba.cut(text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "f9ba0818-d608-4e87-aa9e-aced270a113c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:06.867936Z",
     "iopub.status.busy": "2024-08-15T15:36:06.867647Z",
     "iopub.status.idle": "2024-08-15T15:36:06.873739Z",
     "shell.execute_reply": "2024-08-15T15:36:06.872824Z",
     "shell.execute_reply.started": "2024-08-15T15:36:06.867912Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_emb_retriever(k):\n",
    "    return vector_db.as_retriever(search_kwargs={'k': k})\n",
    "\n",
    "def get_vanilla_bm25_retriever(k):\n",
    "    vanilla_bm25_retriever.k = k\n",
    "    return vanilla_bm25_retriever\n",
    "\n",
    "def get_chz_cut_bm25_retriever(k):\n",
    "    chz_cut_bm25_retriever.k = k\n",
    "    return chz_cut_bm25_retriever\n",
    "\n",
    "def build_get_ensemble_retriver_fn(weights=[0.5, 0.5]):\n",
    "    def get_ensemble_retriever(k):\n",
    "        chz_cut_bm25_retriever.k = k\n",
    "        return EnsembleRetriever(\n",
    "            retrievers=[vector_db.as_retriever(search_kwargs={'k': k}), chz_cut_bm25_retriever], weights=weights\n",
    "        )\n",
    "    return get_ensemble_retriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "b03e3382-39e9-4932-a265-69b811041629",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:07.265370Z",
     "iopub.status.busy": "2024-08-15T15:36:07.264841Z",
     "iopub.status.idle": "2024-08-15T15:36:07.269577Z",
     "shell.execute_reply": "2024-08-15T15:36:07.268795Z",
     "shell.execute_reply.started": "2024-08-15T15:36:07.265345Z"
    }
   },
   "outputs": [],
   "source": [
    "test_df = qa_df[(qa_df['dataset'] == 'test') & (qa_df['qa_type'] == 'detailed')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "32c3ad14-b217-44aa-bdb9-909b9d559668",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:11.591011Z",
     "iopub.status.busy": "2024-08-15T15:36:11.590837Z",
     "iopub.status.idle": "2024-08-15T15:36:11.594260Z",
     "shell.execute_reply": "2024-08-15T15:36:11.593872Z",
     "shell.execute_reply.started": "2024-08-15T15:36:11.590998Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_hit_stat_df(get_retriever_fn, top_k_arr=list(range(1, 9))):\n",
    "    hit_stat_data = []\n",
    "\n",
    "    for k in tqdm(top_k_arr):\n",
    "        retriever = get_retriever_fn(k)\n",
    "        for idx, row in test_df.iterrows():\n",
    "            question = row['question']\n",
    "            true_uuid = row['uuid']\n",
    "            # chunks = retrieve_fn(question, k=k)\n",
    "            chunks = retriever.get_relevant_documents(question)[:k]\n",
    "            retrieved_uuids = [doc.metadata['uuid'] for doc in chunks]\n",
    "\n",
    "            hit_stat_data.append({\n",
    "                'question': question,\n",
    "                'top_k': k,\n",
    "                'hit': int(true_uuid in retrieved_uuids),\n",
    "                'retrieved_chunks': len(chunks)\n",
    "            })\n",
    "    hit_stat_df = pd.DataFrame(hit_stat_data)\n",
    "    return hit_stat_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79482d8c-04e3-4a95-9af7-f54fed7c8d81",
   "metadata": {},
   "source": [
    "## BM25检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "a63797c7-4151-4f55-8e5d-080c34265393",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:12.090218Z",
     "iopub.status.busy": "2024-08-15T15:36:12.089886Z",
     "iopub.status.idle": "2024-08-15T15:36:12.784684Z",
     "shell.execute_reply": "2024-08-15T15:36:12.784263Z",
     "shell.execute_reply.started": "2024-08-15T15:36:12.090189Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c2645d5467a483ea332b44c07094b45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13460ffe3e1449129f86bd0f85a45c76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vanilla_bm25_hit_stat_df = get_hit_stat_df(get_vanilla_bm25_retriever)\n",
    "chz_cut_bm25_hit_stat_df = get_hit_stat_df(get_chz_cut_bm25_retriever)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "11699006-6e3d-4863-a8a9-d7f331571f9f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:14.315836Z",
     "iopub.status.busy": "2024-08-15T15:36:14.315401Z",
     "iopub.status.idle": "2024-08-15T15:36:14.322691Z",
     "shell.execute_reply": "2024-08-15T15:36:14.321903Z",
     "shell.execute_reply.started": "2024-08-15T15:36:14.315798Z"
    }
   },
   "outputs": [],
   "source": [
    "vanilla_bm25_hit_stat_df['retriever'] = 'vanilla_bm25'\n",
    "chz_cut_bm25_hit_stat_df['retriever'] = 'jieba_cut_bm25'\n",
    "\n",
    "bm25_hit_stat_df = pd.concat([vanilla_bm25_hit_stat_df, chz_cut_bm25_hit_stat_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "15c6c97f-e391-4dbc-903d-b2a86035d04f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:14.627836Z",
     "iopub.status.busy": "2024-08-15T15:36:14.627053Z",
     "iopub.status.idle": "2024-08-15T15:36:15.528158Z",
     "shell.execute_reply": "2024-08-15T15:36:15.525706Z",
     "shell.execute_reply.started": "2024-08-15T15:36:14.627765Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>retriever</th>\n",
       "      <th>top_k</th>\n",
       "      <th>hit_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.784946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>3</td>\n",
       "      <td>0.838710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>vanilla_bm25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>vanilla_bm25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>vanilla_bm25</td>\n",
       "      <td>3</td>\n",
       "      <td>0.032258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>vanilla_bm25</td>\n",
       "      <td>4</td>\n",
       "      <td>0.064516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>vanilla_bm25</td>\n",
       "      <td>5</td>\n",
       "      <td>0.075269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>vanilla_bm25</td>\n",
       "      <td>6</td>\n",
       "      <td>0.086022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>vanilla_bm25</td>\n",
       "      <td>7</td>\n",
       "      <td>0.118280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>vanilla_bm25</td>\n",
       "      <td>8</td>\n",
       "      <td>0.118280</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         retriever  top_k  hit_rate\n",
       "0   jieba_cut_bm25      1  0.666667\n",
       "1   jieba_cut_bm25      2  0.784946\n",
       "2   jieba_cut_bm25      3  0.838710\n",
       "3   jieba_cut_bm25      4  0.870968\n",
       "4   jieba_cut_bm25      5  0.892473\n",
       "5   jieba_cut_bm25      6  0.903226\n",
       "6   jieba_cut_bm25      7  0.913978\n",
       "7   jieba_cut_bm25      8  0.935484\n",
       "8     vanilla_bm25      1  0.000000\n",
       "9     vanilla_bm25      2  0.000000\n",
       "10    vanilla_bm25      3  0.032258\n",
       "11    vanilla_bm25      4  0.064516\n",
       "12    vanilla_bm25      5  0.075269\n",
       "13    vanilla_bm25      6  0.086022\n",
       "14    vanilla_bm25      7  0.118280\n",
       "15    vanilla_bm25      8  0.118280"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bm25_hit_stat_df.groupby(['retriever', 'top_k'])['hit'].mean().reset_index().rename(columns={'hit': 'hit_rate'})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ef11c8b-dda2-462d-b53c-6f523ca11c7e",
   "metadata": {},
   "source": [
    "## Embedding检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "0717a80b-37cd-44b4-8db1-e4c917360c83",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:16.245435Z",
     "iopub.status.busy": "2024-08-15T15:36:16.244659Z",
     "iopub.status.idle": "2024-08-15T15:36:34.096826Z",
     "shell.execute_reply": "2024-08-15T15:36:34.096474Z",
     "shell.execute_reply.started": "2024-08-15T15:36:16.245365Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9c4246970404b44b8316744cde99e9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emb_hit_stat_df = get_hit_stat_df(get_emb_retriever)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "a32f1723-4664-4113-9d37-bde97c56ec3a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:34.097568Z",
     "iopub.status.busy": "2024-08-15T15:36:34.097431Z",
     "iopub.status.idle": "2024-08-15T15:36:34.099878Z",
     "shell.execute_reply": "2024-08-15T15:36:34.099555Z",
     "shell.execute_reply.started": "2024-08-15T15:36:34.097555Z"
    }
   },
   "outputs": [],
   "source": [
    "emb_hit_stat_df['retriever'] = 'embedding'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f252c685-6ce9-423b-bb1c-7a318a145351",
   "metadata": {},
   "source": [
    "## Ensemble检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "b6917c95-96a5-4efc-beef-dfffb2ef1dc8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:36:54.514426Z",
     "iopub.status.busy": "2024-08-15T15:36:54.512048Z",
     "iopub.status.idle": "2024-08-15T15:38:08.862086Z",
     "shell.execute_reply": "2024-08-15T15:38:08.861683Z",
     "shell.execute_reply.started": "2024-08-15T15:36:54.514344Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "253c63ef1ecf41cbb7dea08713b7c745",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f553c2afaeb144d6a90e12b646f16718",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "191ff4bc8dc4463eae3dd3647005e885",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1340d89eb5344a7b17a522861e71e9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ensemble_hit_stat_df_04_06 = get_hit_stat_df(build_get_ensemble_retriver_fn(weights=[0.4, 0.6]))\n",
    "ensemble_hit_stat_df_05_05 = get_hit_stat_df(build_get_ensemble_retriver_fn(weights=[0.5, 0.5]))\n",
    "ensemble_hit_stat_df_07_03 = get_hit_stat_df(build_get_ensemble_retriver_fn(weights=[0.7, 0.3]))\n",
    "ensemble_hit_stat_df_09_01 = get_hit_stat_df(build_get_ensemble_retriver_fn(weights=[0.9, 0.1]))\n",
    "\n",
    "ensemble_hit_stat_df_04_06['retriever'] = 'ensemble_0.4:0.6'\n",
    "ensemble_hit_stat_df_05_05['retriever'] = 'ensemble_0.5:0.5'\n",
    "ensemble_hit_stat_df_07_03['retriever'] = 'ensemble_0.7:0.3'\n",
    "ensemble_hit_stat_df_09_01['retriever'] = 'ensemble_0.90.1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "4a5c1b5f-5363-4304-950b-511e4debf4bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:38:08.862893Z",
     "iopub.status.busy": "2024-08-15T15:38:08.862710Z",
     "iopub.status.idle": "2024-08-15T15:38:08.865755Z",
     "shell.execute_reply": "2024-08-15T15:38:08.865404Z",
     "shell.execute_reply.started": "2024-08-15T15:38:08.862880Z"
    }
   },
   "outputs": [],
   "source": [
    "ensemble_hit_stat_df = pd.concat([ensemble_hit_stat_df_04_06, ensemble_hit_stat_df_05_05, ensemble_hit_stat_df_07_03, ensemble_hit_stat_df_09_01])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "982c515f-d964-47f8-8a9e-2ce277fb42c1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:38:08.866359Z",
     "iopub.status.busy": "2024-08-15T15:38:08.866240Z",
     "iopub.status.idle": "2024-08-15T15:38:08.884271Z",
     "shell.execute_reply": "2024-08-15T15:38:08.883841Z",
     "shell.execute_reply.started": "2024-08-15T15:38:08.866347Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>retriever</th>\n",
       "      <th>top_k</th>\n",
       "      <th>hit_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>1</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0.784946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>3</td>\n",
       "      <td>0.838710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>0.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>2</td>\n",
       "      <td>0.774194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>3</td>\n",
       "      <td>0.860215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>4</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>5</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.946237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>7</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>8</td>\n",
       "      <td>0.978495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>2</td>\n",
       "      <td>0.698925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.698925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           retriever  top_k  hit_rate\n",
       "0   ensemble_0.4:0.6      1  0.666667\n",
       "1   ensemble_0.4:0.6      2  0.784946\n",
       "2   ensemble_0.4:0.6      3  0.838710\n",
       "3   ensemble_0.4:0.6      4  0.870968\n",
       "4   ensemble_0.4:0.6      5  0.892473\n",
       "5   ensemble_0.4:0.6      6  0.903226\n",
       "6   ensemble_0.4:0.6      7  0.913978\n",
       "7   ensemble_0.4:0.6      8  0.935484\n",
       "8   ensemble_0.5:0.5      1  0.548387\n",
       "9   ensemble_0.5:0.5      2  0.774194\n",
       "10  ensemble_0.5:0.5      3  0.860215\n",
       "11  ensemble_0.5:0.5      4  0.892473\n",
       "12  ensemble_0.5:0.5      5  0.913978\n",
       "13  ensemble_0.5:0.5      6  0.946237\n",
       "14  ensemble_0.5:0.5      7  0.956989\n",
       "15  ensemble_0.5:0.5      8  0.978495\n",
       "16  ensemble_0.7:0.3      1  0.548387\n",
       "17  ensemble_0.7:0.3      2  0.698925\n",
       "18  ensemble_0.7:0.3      3  0.806452\n",
       "19  ensemble_0.7:0.3      4  0.870968\n",
       "20  ensemble_0.7:0.3      5  0.892473\n",
       "21  ensemble_0.7:0.3      6  0.903226\n",
       "22  ensemble_0.7:0.3      7  0.913978\n",
       "23  ensemble_0.7:0.3      8  0.935484\n",
       "24   ensemble_0.90.1      1  0.548387\n",
       "25   ensemble_0.90.1      2  0.698925\n",
       "26   ensemble_0.90.1      3  0.806452\n",
       "27   ensemble_0.90.1      4  0.870968\n",
       "28   ensemble_0.90.1      5  0.892473\n",
       "29   ensemble_0.90.1      6  0.903226\n",
       "30   ensemble_0.90.1      7  0.913978\n",
       "31   ensemble_0.90.1      8  0.935484"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ensemble_hit_stat_df.groupby(['retriever', 'top_k'])['hit'].mean().reset_index().rename(columns={'hit': 'hit_rate'})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c729e9c5-650f-4815-9a31-ed6ceeff2e82",
   "metadata": {},
   "source": [
    "## 综合对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "d4890789-a44c-41de-b17f-0ff505788494",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:38:08.885142Z",
     "iopub.status.busy": "2024-08-15T15:38:08.885014Z",
     "iopub.status.idle": "2024-08-15T15:38:08.887886Z",
     "shell.execute_reply": "2024-08-15T15:38:08.887514Z",
     "shell.execute_reply.started": "2024-08-15T15:38:08.885130Z"
    }
   },
   "outputs": [],
   "source": [
    "hit_stat_df = pd.concat([chz_cut_bm25_hit_stat_df, emb_hit_stat_df, ensemble_hit_stat_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "23c07eaf-889c-4b13-83bd-34d75fd98e24",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:38:08.888448Z",
     "iopub.status.busy": "2024-08-15T15:38:08.888325Z",
     "iopub.status.idle": "2024-08-15T15:38:08.898115Z",
     "shell.execute_reply": "2024-08-15T15:38:08.897794Z",
     "shell.execute_reply.started": "2024-08-15T15:38:08.888436Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>retriever</th>\n",
       "      <th>top_k</th>\n",
       "      <th>hit_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>embedding</td>\n",
       "      <td>1</td>\n",
       "      <td>0.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>embedding</td>\n",
       "      <td>2</td>\n",
       "      <td>0.698925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>embedding</td>\n",
       "      <td>3</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>embedding</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>embedding</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>embedding</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>embedding</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>embedding</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>1</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>2</td>\n",
       "      <td>0.784946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>3</td>\n",
       "      <td>0.838710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ensemble_0.4:0.6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>1</td>\n",
       "      <td>0.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>2</td>\n",
       "      <td>0.774194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>3</td>\n",
       "      <td>0.860215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>4</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>5</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>6</td>\n",
       "      <td>0.946237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>7</td>\n",
       "      <td>0.956989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ensemble_0.5:0.5</td>\n",
       "      <td>8</td>\n",
       "      <td>0.978495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>1</td>\n",
       "      <td>0.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>2</td>\n",
       "      <td>0.698925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>ensemble_0.7:0.3</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.698925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.806452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>ensemble_0.90.1</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>1</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>2</td>\n",
       "      <td>0.784946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>3</td>\n",
       "      <td>0.838710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>4</td>\n",
       "      <td>0.870968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>5</td>\n",
       "      <td>0.892473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>6</td>\n",
       "      <td>0.903226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>7</td>\n",
       "      <td>0.913978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>jieba_cut_bm25</td>\n",
       "      <td>8</td>\n",
       "      <td>0.935484</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           retriever  top_k  hit_rate\n",
       "0          embedding      1  0.548387\n",
       "1          embedding      2  0.698925\n",
       "2          embedding      3  0.806452\n",
       "3          embedding      4  0.870968\n",
       "4          embedding      5  0.892473\n",
       "5          embedding      6  0.903226\n",
       "6          embedding      7  0.913978\n",
       "7          embedding      8  0.935484\n",
       "8   ensemble_0.4:0.6      1  0.666667\n",
       "9   ensemble_0.4:0.6      2  0.784946\n",
       "10  ensemble_0.4:0.6      3  0.838710\n",
       "11  ensemble_0.4:0.6      4  0.870968\n",
       "12  ensemble_0.4:0.6      5  0.892473\n",
       "13  ensemble_0.4:0.6      6  0.903226\n",
       "14  ensemble_0.4:0.6      7  0.913978\n",
       "15  ensemble_0.4:0.6      8  0.935484\n",
       "16  ensemble_0.5:0.5      1  0.548387\n",
       "17  ensemble_0.5:0.5      2  0.774194\n",
       "18  ensemble_0.5:0.5      3  0.860215\n",
       "19  ensemble_0.5:0.5      4  0.892473\n",
       "20  ensemble_0.5:0.5      5  0.913978\n",
       "21  ensemble_0.5:0.5      6  0.946237\n",
       "22  ensemble_0.5:0.5      7  0.956989\n",
       "23  ensemble_0.5:0.5      8  0.978495\n",
       "24  ensemble_0.7:0.3      1  0.548387\n",
       "25  ensemble_0.7:0.3      2  0.698925\n",
       "26  ensemble_0.7:0.3      3  0.806452\n",
       "27  ensemble_0.7:0.3      4  0.870968\n",
       "28  ensemble_0.7:0.3      5  0.892473\n",
       "29  ensemble_0.7:0.3      6  0.903226\n",
       "30  ensemble_0.7:0.3      7  0.913978\n",
       "31  ensemble_0.7:0.3      8  0.935484\n",
       "32   ensemble_0.90.1      1  0.548387\n",
       "33   ensemble_0.90.1      2  0.698925\n",
       "34   ensemble_0.90.1      3  0.806452\n",
       "35   ensemble_0.90.1      4  0.870968\n",
       "36   ensemble_0.90.1      5  0.892473\n",
       "37   ensemble_0.90.1      6  0.903226\n",
       "38   ensemble_0.90.1      7  0.913978\n",
       "39   ensemble_0.90.1      8  0.935484\n",
       "40    jieba_cut_bm25      1  0.666667\n",
       "41    jieba_cut_bm25      2  0.784946\n",
       "42    jieba_cut_bm25      3  0.838710\n",
       "43    jieba_cut_bm25      4  0.870968\n",
       "44    jieba_cut_bm25      5  0.892473\n",
       "45    jieba_cut_bm25      6  0.903226\n",
       "46    jieba_cut_bm25      7  0.913978\n",
       "47    jieba_cut_bm25      8  0.935484"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hit_stat_df.groupby(['retriever', 'top_k'])['hit'].mean().reset_index().rename(columns={'hit': 'hit_rate'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "b0b086d1-6cec-4743-8df6-2ab3b1593689",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-15T15:38:08.898694Z",
     "iopub.status.busy": "2024-08-15T15:38:08.898573Z",
     "iopub.status.idle": "2024-08-15T15:38:09.094618Z",
     "shell.execute_reply": "2024-08-15T15:38:09.094192Z",
     "shell.execute_reply.started": "2024-08-15T15:38:08.898682Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='top_k', ylabel='hit'>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIQAAAINCAYAAABPvaQcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABlQUlEQVR4nO3deVyVdd7/8fcBlEUEIhVEUXALLBAVRVxGGzE1x9E0c6EQt2omFCXNXTEVbNHcSs3dGcmWu7ydsZyUBg0zFddKhxL1xkxBMzRFUTj8/vDnmU4sYpIHuV7Px+M8Hpzr+i6f69zXzD29+17fy1RYWFgoAAAAAAAAGIadrQsAAAAAAADAvUUgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAOti7gXjObzfrhhx9UvXp1mUwmW5cDAAAAAABQLgoLC/Xzzz/Lx8dHdnalrwEyXCD0ww8/yNfX19ZlAAAAAAAA/C5OnTqlunXrltrGcIFQ9erVJd38cdzc3GxcDQAAAAAAQPm4dOmSfH19LdlHaQwXCN16TMzNzY1ACAAAAAAAVDpl2SKHTaUBAAAAAAAMhkAIAAAAAADAYAiEAAAAAAAADMZwewiVRWFhofLz81VQUGDrUoBSValSRfb29rYuAwAAAABwnyEQ+pXr16/rzJkzys3NtXUpwG2ZTCbVrVtXrq6uti4FAAAAAHAfsWkgtGPHDr322mvat2+fzpw5o48++ki9e/cutU9KSori4uL0zTffyNfXV1OmTFF0dHS51GM2m3XixAnZ29vLx8dHVatWLdPO3IAtFBYW6ty5c/r+++/VuHFjVgoBAAAAAMrMpoHQlStX1KxZMw0dOlR9+vS5bfsTJ06oR48eev7557V+/XolJydr+PDhql27trp27XrX9Vy/fl1ms1m+vr5ycXG56/GA31vNmjV18uRJ3bhxg0AIAAAAAFBmNg2Eunfvru7du5e5/dKlS+Xv76+5c+dKkgIDA5Wamqo33nijXAKhW+zs2Gsb9wdWsAEAAAAAfov7KvnYtWuXIiIirI517dpVu3btKrFPXl6eLl26ZPUBAAAAAAAwsvsqEDp79qy8vLysjnl5eenSpUu6evVqsX0SExPl7u5u+fj6+t6LUgEAAAAAACqs+yoQ+i0mTpyoixcvWj6nTp2ydUmGEB8fr5CQEFuXAQAAAAAAinFfBULe3t7KysqyOpaVlSU3Nzc5OzsX28fR0VFubm5WH/w2nTp10ujRo8vUduzYsUpOTv59CwIAAAAAAL/JfRUIhYeHFwkZtm7dqvDwcBtVVHlcv369XMYpLCxUfn6+XF1d9eCDD5bLmL9VQUGBzGazTWsAAAAAAKAismkgdPnyZR08eFAHDx6UdPO18gcPHlRmZqakm497RUVFWdo///zzOn78uF566SX95z//0VtvvaX33ntPY8aMsUX597VOnTopJiZGo0ePVo0aNdS1a1d9/fXX6t69u1xdXeXl5aVnnnlG58+flyRFR0dr+/btWrBggUwmk0wmk06ePKmUlBSZTCZ98sknatmypRwdHZWamlrsI2MrVqxQYGCgnJycFBAQoLfeestyrm3btho/frxV+3PnzqlKlSrasWOHpJsbhI8dO1Z16tRRtWrVFBYWppSUFEv7NWvWyMPDQ5s2bVLTpk3l6OhouZcAAAAAAMB/2TQQSktLU/PmzdW8eXNJUlxcnJo3b65p06ZJks6cOWP1D/T+/v7avHmztm7dqmbNmmnu3LlasWJFub5y3kjWrl2rqlWraufOnZozZ47++Mc/qnnz5kpLS9OWLVuUlZWlp556SpK0YMEChYeHa8SIETpz5ozOnDljtUH3hAkTNGfOHB09elTBwcFF5lq/fr2mTZum2bNn6+jRo0pISNDUqVO1du1aSVJkZKQ2bNigwsJCS593331XPj4+6tChgyQpJiZGu3bt0oYNG3T48GH169dP3bp103fffWfpk5ubq1deeUUrVqzQN998o1q1av0uvx0AAAAAAPczB1tO3qlTJ6sA4NfWrFlTbJ8DBw78jlUZR+PGjfXqq69KkmbNmqXmzZsrISHBcn7VqlXy9fXVt99+qyZNmqhq1apycXGRt7d3kbFefvlldenSpcS5pk+frrlz56pPnz6SboZ7R44c0bJlyzR48GA99dRTGj16tFJTUy0BUFJSkgYOHCiTyaTMzEytXr1amZmZ8vHxkXRzn6ItW7Zo9erVlrpv3Liht956S82aNSufHwkAAAAAgErIpoEQbKtly5aWvw8dOqR///vfcnV1LdIuIyNDTZo0KXWs0NDQEs9duXJFGRkZGjZsmEaMGGE5np+fL3d3d0lSzZo19dhjj2n9+vXq0KGDTpw4oV27dmnZsmWSpK+++koFBQVF6sjLy7Paq6hq1arFrlACAAAAAAD/RSBkYNWqVbP8ffnyZfXs2VOvvPJKkXa1a9e+o7F+7fLly5Kk5cuXKywszOqcvb295e/IyEiNGjVKixYtUlJSkoKCghQUFGQZw97eXvv27bPqI8kqxHJ2dpbJZLptvQAAAAAAGBmBECRJLVq00P/8z//Iz89PDg7F3xZVq1ZVQUHBHY/t5eUlHx8fHT9+XJGRkSW269Wrl5599llt2bJFSUlJVhuKN2/eXAUFBcrOzrY8UgYAAAAAAH4bAiFIkl544QUtX75cAwcO1EsvvSRPT08dO3ZMGzZs0IoVK2Rvby8/Pz/t3r1bJ0+elKurqzw9Pcs8/owZMzRq1Ci5u7urW7duysvLU1pamn766SfFxcVJurnKqHfv3po6daqOHj2qgQMHWvo3adJEkZGRioqK0ty5c9W8eXOdO3dOycnJCg4OVo8ePcr9NwEAAAAAVHzb/9Cx3MbquGN7uY1V0dn0LWOoOHx8fLRz504VFBToscceU1BQkEaPHi0PDw/Z2d28TcaOHSt7e3s1bdpUNWvWvKNXug8fPlwrVqzQ6tWrFRQUpI4dO2rNmjXy9/e3ahcZGalDhw6pQ4cOqlevntW51atXKyoqSi+++KIeeugh9e7dW3v37i3SDgAAAAAAlM5UWNprviqhS5cuyd3dXRcvXpSbm5vVuWvXrunEiRPy9/eXk5OTjSoEyo57FgAAAIDRsULov0rLPH6NFUIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwDrYuAAAAAACA2ymvV4vf768VB8oLK4QAAAAAAAAMhkAIAAAAAADAYHhkrIxajlt3T+fb91rUHbWPjo5WTk6ONm7cqE6dOikkJETz588vU981a9Zo9OjRysnJufNCKzCTyaSPPvpIvXv3tnUpAAAAAABUKKwQqiQWLFigNWvWSJI+/PBDzZw507YFlbP4+HiFhITYugxJ0vLly9WhQwc98MADeuCBBxQREaE9e/ZYtYmOjpbJZLL6dOvWzUYVAwAAAABgjRVClYS7u7vlb09PTxtWUvmlpKRo4MCBatu2rZycnPTKK6/oscce0zfffKM6depY2nXr1k2rV6+2fHd0dLRFuQAAAPcNNg0GgHuHFUKVRHR0tOXRqE6dOmn06NGWc3l5eRo7dqzq1KmjatWqKSwsTCkpKUXG2Lhxoxo3biwnJyd17dpVp06dspzLyMhQr1695OXlJVdXV7Vq1Urbtm0rc315eXkaP368fH195ejoqEaNGmnlypWSbj6y5uHhUaQWk8lkOT9jxgwdOnTIstrm1mqo2zlz5oy6d+8uZ2dnNWjQQB988IHl3MmTJ2UymfTee++pQ4cOcnZ2VqtWrfTtt99q7969Cg0Nlaurq7p3765z585Z+q1fv15//etfFRISooCAAK1YsUJms1nJyclWczs6Osrb29vyeeCBB8r8ewEAAAAA8HsiEDKAmJgY7dq1Sxs2bNDhw4fVr18/devWTd99952lTW5urmbPnq1169Zp586dysnJ0YABAyznL1++rMcff1zJyck6cOCAunXrpp49eyozM7NMNURFRemdd97RwoULdfToUS1btkyurq5l6tu/f3+9+OKLevjhh3XmzBmdOXNG/fv3L1PfqVOnqm/fvjp06JAiIyM1YMAAHT161KrN9OnTNWXKFO3fv18ODg4aNGiQXnrpJS1YsECff/65jh07pmnTppU4R25urm7cuFFkZVZKSopq1aqlhx56SH/5y1/0448/lqlmAAAAAAB+bzwyVsllZmZq9erVyszMlI+PjyRp7Nix2rJli1avXq2EhARJ0o0bN7R48WKFhYVJktauXavAwEDt2bNHrVu3VrNmzdSsWTPLuDNnztRHH32kTZs2KSYmptQavv32W7333nvaunWrIiIiJEkNGjQo8zU4OzvL1dVVDg4O8vb2vqPr79evn4YPH26peevWrVq0aJHeeustS5uxY8eqa9eukqTY2FgNHDhQycnJateunSRp2LBhpa5IGj9+vHx8fCzXJt18XKxPnz7y9/dXRkaGJk2apO7du2vXrl2yt7e/o2sAAAAAAKC8EQhVcl999ZUKCgrUpEkTq+N5eXl68MEHLd8dHBzUqlUry/eAgAB5eHjo6NGjat26tS5fvqz4+Hht3rxZZ86cUX5+vq5evVqmFUIHDx6Uvb29OnYsn2fC70R4eHiR7wcPHrQ6FhwcbPnby8tLkhQUFGR1LDs7u9jx58yZow0bNiglJUVOTk6W479cXRUUFKTg4GA1bNhQKSkp6ty582++HgAAAAAAygOBUCV3+fJl2dvba9++fUVWppT1kS3p5iqarVu36vXXX1ejRo3k7OysJ598UtevX79tX2dn51LP29nZqbCw0OrYjRs3ylzb3apSpYrl71v7Fv36mNlsLtLv9ddf15w5c7Rt2zarUKk4DRo0UI0aNXTs2DECIQAAAAA20XLcunIZ56Pqr5XLOAMfcCuXcRLKMdpY/OI/ymWci1lry2WcyX//4PaNfiP2EKrkmjdvroKCAmVnZ6tRo0ZWn18+fpWfn6+0tDTL9/T0dOXk5CgwMFCStHPnTkVHR+uJJ55QUFCQvL29dfLkyTLVEBQUJLPZrO3bi3/bQ82aNfXzzz/rypUrlmO/XsVTtWpVFRQUlPGq/+vLL78s8v3WNd2NV199VTNnztSWLVsUGhp62/bff/+9fvzxR9WuXfuu5wYAAAAA4G4RCFVyTZo0UWRkpKKiovThhx/qxIkT2rNnjxITE7V582ZLuypVqmjkyJHavXu39u3bp+joaLVp00atW7eWJDVu3FgffvihDh48qEOHDmnQoEHFrpopjp+fnwYPHqyhQ4dq48aNOnHihFJSUvTee+9JksLCwuTi4qJJkyYpIyNDSUlJRfbs8fPz04kTJ3Tw4EGdP39eeXl5ZZr7/fff16pVq/Ttt99q+vTp2rNnz233PLqdV155RVOnTtWqVavk5+ens2fP6uzZs7p8+bKkm6uyxo0bpy+//FInT55UcnKyevXqpUaNGln2KgIAAAAAwJZ4ZKyM9r0WZesSfrPVq1dr1qxZevHFF3X69GnVqFFDbdq00Z/+9CdLGxcXF40fP16DBg3S6dOn1aFDB8tr4SVp3rx5Gjp0qNq2basaNWpo/PjxunTpUplrWLJkiSZNmqS//vWv+vHHH1WvXj1NmjRJkuTp6am///3vGjdunJYvX67OnTsrPj5ezz77rKV/37599eGHH+rRRx9VTk6OVq9erejo6NvOO2PGDG3YsEF//etfVbt2bb3zzjtq2rRpmesu6VquX7+uJ5980ur49OnTFR8fL3t7ex0+fFhr165VTk6OfHx89Nhjj2nmzJlydHS8q7kBAAAAACgPpsJfb95SyV26dEnu7u66ePGi3Nysn1e8du2aTpw4IX9/f6sNgu8HAwcOlL29vf7+97/buhTcQ/fzPQsAAPBr2/9QPi8h6bij+K0KcH/j/rh7lXYPoffLb63LV63Glss4ttpDqLTM49d4ZOw+l5+fryNHjmjXrl16+OGHbV0OAAAAAAC4DxAI3ee+/vprhYaG6uGHH9bzzz9vkxo+//xzubq6lvj5Paxfv77E+QjGAAAAAAAoHXsI3edCQkKUm5tr0xpCQ0OLvBXs9/bnP/9ZYWFhxZ775SvjAQAAAABAUQRCuGvOzs5q1KjRPZ2zevXqql69+j2dEwAAAACAyoJHxgAAAAAAAAyGFUIAAAC4p3hTEPD7qrRvkiqnf3xd/OI/ymUcW71FCigvrBACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBj2ECqjzJeD7ul89aZ9dU/nK06nTp0UEhKi+fPnl+u4a9as0ejRo5WTk1Nim/j4eG3cuNHyOvvo6Gjl5ORo48aN5VoLAADA/Yg9YkrHHjEAcHsEQrgvLFiwQIWFhbYuAwAAAACASoFACPcFd3d3W5cAAAAAAEClwR5ClYTZbFZiYqL8/f3l7OysZs2a6YMPbi5NTUlJkclk0r/+9S81b95czs7O+uMf/6js7Gx98sknCgwMlJubmwYNGqTc3FyrcfPz8xUTEyN3d3fVqFFDU6dOtVqpk5eXp7Fjx6pOnTqqVq2awsLClJKSYjXGmjVrVK9ePbm4uOiJJ57Qjz/+WKT+OXPmyMvLS9WrV9ewYcN07do1q/PR0dHq3bu35XunTp00atQovfTSS/L09JS3t7fi4+Ot+vznP/9R+/bt5eTkpKZNm2rbtm0ymUw8dgYAAAAAMDwCoUoiMTFR69at09KlS/XNN99ozJgxevrpp7V9+3ZLm/j4eC1evFhffPGFTp06paeeekrz589XUlKSNm/erE8//VSLFi2yGnft2rVycHDQnj17tGDBAs2bN08rVqywnI+JidGuXbu0YcMGHT58WP369VO3bt303XffSZJ2796tYcOGKSYmRgcPHtSjjz6qWbNmWc3x3nvvKT4+XgkJCUpLS1Pt2rX11ltv3faa165dq2rVqmn37t169dVX9fLLL2vr1q2SpIKCAvXu3VsuLi7avXu33n77bU2ePPk3/74AAAAAAFQmPDJWCeTl5SkhIUHbtm1TeHi4JKlBgwZKTU3VsmXL9Oyzz0qSZs2apXbt2kmShg0bpokTJyojI0MNGjSQJD355JP697//rfHjx1vG9vX11RtvvCGTyaSHHnpIX331ld544w2NGDFCmZmZWr16tTIzM+Xj4yNJGjt2rLZs2aLVq1crISFBCxYsULdu3fTSSy9Jkpo0aaIvvvhCW7Zsscwxf/58DRs2TMOGDbPUuW3btiKrhH4tODhY06dPlyQ1btxYixcvVnJysrp06aKtW7cqIyNDKSkp8vb2liTNnj1bXbp0ubsfGwBQJtv/0LFcxum4Y/vtG1VSFW3TYImNg2+HjYMBAPcTVghVAseOHVNubq66dOkiV1dXy2fdunXKyMiwtAsODrb87eXlJRcXF0sYdOtYdna21dht2rSRyWSyfA8PD9d3332ngoICffXVVyooKFCTJk2s5t2+fbtl3qNHjyosLMxqzFuh1S1laVOcX16PJNWuXdtSf3p6unx9fS1hkCS1bt36tmMCAAAAAGAErBCqBC5fvixJ2rx5s+rUqWN1ztHR0RLOVKlSxXLcZDJZfb91zGw239G89vb22rdvn+zt7a3Oubq63tE1/BZ3Wz8AAAAAAEZFIFQJNG3aVI6OjsrMzFTHjkWX6P9yldCd2r17t9X3L7/8Uo0bN5a9vb2aN2+ugoICZWdnq0OHDsX2DwwMLHaM4tpERUWV2OZOPfTQQzp16pSysrLk5eUlSdq7d+9djQkAAAAAQGVBIFQJVK9eXWPHjtWYMWNkNpvVvn17Xbx4UTt37pSbm5vq16//m8fOzMxUXFycnnvuOe3fv1+LFi3S3LlzJd3cDygyMlJRUVGaO3eumjdvrnPnzik5OVnBwcHq0aOHRo0apXbt2un1119Xr1699K9//ctq/yBJio2NVXR0tEJDQ9WuXTutX79e33zzjdXjbHeqS5cuatiwoQYPHqxXX31VP//8s6ZMmSJJVo/AAQAAAABgRARCZVRv2le2LqFUM2fOVM2aNZWYmKjjx4/Lw8NDLVq00KRJk+7qMaqoqChdvXpVrVu3lr29vWJjYy2bVEvS6tWrNWvWLL344os6ffq0atSooTZt2uhPf/qTpJt7EC1fvlzTp0/XtGnTFBERoSlTpmjmzJmWMfr376+MjAy99NJLunbtmvr27au//OUv+te//vWb67a3t9fGjRs1fPhwtWrVSg0aNNBrr72mnj17ysnJ6TePCwAAAABAZUAgVEmYTCbFxsYqNja22POFhYVW36OjoxUdHW11LD4+XvHx8ZbvKSkplr+XLFlS7LhVqlTRjBkzNGPGjBJrGzp0qIYOHWp17MUXX7T6PmnSJE2aNMnq2CuvvGL5e82aNVbnflnbLRs3brT6HhAQoNTUVMv3nTt3SpIaNWpUYq0AAAAAABgBgRAqrY8++kiurq5q3Lixjh07ptjYWLVr104NGza0dWkAAAAAANgUgRAqrZ9//lnjx49XZmamatSooYiICMv+RwAAAAAAGBmBECqtqKgoqzeXAQAAAACAmwiEAACwoZbj1pXLOB9Vf61cxhn4gFu5jJNQTv8TY/GL/yiXcS5mrS2XcSb//YNyGQcAAMDW7GxdAAAAAAAAAO4tAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAg2FTaQDAb7L9Dx3LZZyOO7aXyzgAAAAAyo5AqIzaLWp3T+fbOXLnPZ3PFtasWaPRo0crJyenxDbx8fHauHGjDh48eM/qAgAAAACgsuORMRjKhQsXFBkZKTc3N3l4eGjYsGG6fPlymfoWFhaqe/fuMplM2rhx423bv//++woICJCTk5OCgoL08ccf37ZPXl6eJk+erPr168vR0VF+fn5atWpVmeoDAAAAAKCsWCEEQ4mMjNSZM2e0detW3bhxQ0OGDNGzzz6rpKSk2/adP3++TCZTmeb54osvNHDgQCUmJupPf/qTkpKS1Lt3b+3fv1+PPPJIif2eeuopZWVlaeXKlWrUqJHOnDkjs9lc5usDAAAAAKAsWCFUSZjNZiUmJsrf31/Ozs5q1qyZPvjgA0lSSkqKTCaTkpOTFRoaKhcXF7Vt21bp6emW/ocOHdKjjz6q6tWry83NTS1btlRaWprlfGpqqjp06CBnZ2f5+vpq1KhRunLliuW8n5+fZs2apaioKLm6uqp+/fratGmTzp07p169esnV1VXBwcFWY96yceNGNW7cWE5OTuratatOnTpV6rWuWLFCgYGBcnJyUkBAgN56660y/UZHjx7Vli1btGLFCoWFhal9+/ZatGiRNmzYoB9++KHUvgcPHtTcuXPLvFpnwYIF6tatm8aNG6fAwEDNnDlTLVq00OLFi0vss2XLFm3fvl0ff/yxIiIi5Ofnp/DwcLVrd28fVwQAAAAAVH4EQpVEYmKi1q1bp6VLl+qbb77RmDFj9PTTT2v79v9u1jp58mTNnTtXaWlpcnBw0NChQy3nIiMjVbduXe3du1f79u3ThAkTVKVKFUlSRkaGunXrpr59++rw4cN69913lZqaqpiYGKsa3njjDbVr104HDhxQjx499MwzzygqKkpPP/209u/fr4YNGyoqKkqFhYWWPrm5uZo9e7bWrVunnTt3KicnRwMGDCjxOtevX69p06Zp9uzZOnr0qBISEjR16lStXbv2tr/Rrl275OHhodDQUMuxiIgI2dnZaffu3SX2y83N1aBBg/Tmm2/K29u72DZ+fn6Kj4+3misiIsKqTdeuXbVr164S59m0aZNCQ0P16quvqk6dOmrSpInGjh2rq1ev3vbaAAAAAAC4EzwyVgnk5eUpISFB27ZtU3h4uCSpQYMGSk1N1bJly/Tss89KkmbPnq2OHW++FWjChAnq0aOHrl27JicnJ2VmZmrcuHEKCAiQJDVu3NgyfmJioiIjIzV69GjLuYULF6pjx45asmSJnJycJEmPP/64nnvuOUnStGnTtGTJErVq1Ur9+vWTJI0fP17h4eHKysqyBCs3btzQ4sWLFRYWJklau3atAgMDtWfPHrVu3brItU6fPl1z585Vnz59JEn+/v46cuSIli1bpsGDB5f6O509e1a1atWyOubg4CBPT0+dPXu2xH5jxoxR27Zt1atXrxLbNGzYUDVq1LCay8vLy6qNl5dXqfMcP35cqampcnJy0kcffaTz58/rr3/9q3788UetXr261GsDAAAAAOBOEAhVAseOHVNubq66dOlidfz69etq3ry55XtwcLDl79q1a0uSsrOzVa9ePcXFxWn48OH629/+poiICPXr108NGzaUdPNxssOHD2v9+vWW/oWFhTKbzTpx4oQCAwOLjH8rDAkKCipyLDs72xIIOTg4qFWrVpY2AQEB8vDw0NGjR4sEQleuXFFGRoaGDRumESNGWI7n5+fL3d29zL/Xndi0aZM+++wzHThwoNR2ycnJdz2X2WyWyWTS+vXrLdczb948Pfnkk3rrrbfk7Ox813MAAAAAACARCFUKt96StXnzZtWpU8fqnKOjozIyMiTJ8giYJMvmyLc2LI6Pj9egQYO0efNmffLJJ5o+fbo2bNigJ554QpcvX9Zzzz2nUaNGFZm7Xr16lr+LG7+0OX/rdS5fvtyyougWe3v72/b39vZWdna21bH8/HxduHChxEfBPvvsM2VkZMjDw8PqeN++fdWhQwelpKSUOFdWVpbVsV+ujCpO7dq1VadOHatwKzAwUIWFhfr++++tVm0BAAAAAHA3CIQqgaZNm8rR0VGZmZmWR8J+6VYgdDtNmjRRkyZNNGbMGA0cOFCrV6/WE088oRYtWujIkSNq1KhReZeu/Px8paWlWVYDpaenKycnx7Lq6Je8vLzk4+Oj48ePKzIy8o7nCg8PV05Ojvbt26eWLVtKuhn4mM3mIgHTLRMmTNDw4cOtjgUFBemNN95Qz549S50rOTnZ8pidJG3dutXySF9x2rVrp/fff1+XL1+Wq6urJOnbb7+VnZ2d6tatW9bLRAXUcty6chln32tR5TJOu0Xls1F5Av8vBAAAALhv8b/mK4Hq1atr7NixGjNmjMxms9q3b6+LFy9q586dcnNzU/369Uvtf/XqVY0bN05PPvmk/P399f3332vv3r3q27evpJt7/7Rp00YxMTEaPny4qlWrpiNHjmjr1q2lvjWrLKpUqaKRI0dq4cKFcnBwUExMjNq0aVPs/kGSNGPGDI0aNUru7u7q1q2b8vLylJaWpp9++klxcXGlzhUYGKhu3bppxIgRWrp0qW7cuKGYmBgNGDBAPj4+kqTTp0+rc+fOWrdunVq3bi1vb+9iV/XUq1dP/v7+lu+dO3fWE088YdloOzY2Vh07dtTcuXPVo0cPbdiwQWlpaXr77bctfSZOnKjTp09r3bqbYcGgQYM0c+ZMDRkyRDNmzND58+c1btw4DR06lMfFAAAAAADlikCojHaO3GnrEko1c+ZM1axZU4mJiTp+/Lg8PDzUokULTZo06baPaNnb2+vHH39UVFSUsrKyVKNGDfXp00czZsyQdHNvoO3bt2vy5Mnq0KGDCgsL1bBhQ/Xv3/+u63ZxcdH48eM1aNAgnT59Wh06dNDKlStLbD98+HC5uLjotdde07hx41StWjUFBQVZrcQpzfr16xUTE6POnTvLzs5Offv21cKFCy3nb9y4ofT0dOXm5t7RdWRkZOj8+fOW723btlVSUpKmTJmiSZMmqXHjxtq4caMeeeQRS5szZ84oMzPT8t3V1VVbt27VyJEjFRoaqgcffFBPPfWUZs2adUe1AAAAAABwOwRClYTJZFJsbKxiY2OLPf/LV71LUkhIiNWxd955p9TxW7VqpU8//bTE8ydPnrztnH5+flbHoqOjFR0dLUmWt4b9Wnx8vNXr3KWbK2kGDRpUar0l8fT0VFJSUonnf11jcYo7X9z19+vXz/KGteKsWbOmyLGAgABt3bq11PkBAAAAALhbdrYuAAAAAAAAAPcWgRAqjYSEBLm6uhb76d69u63LAwAAAACgwuCRMVQazz//vJ566qliz7EpMwAAAAAA/0UghErD09NTnp6eti4DAAAAAIAKj0fGAAAAAAAADIZACAAAAAAAwGAIhAAAAAAAAAyGQAgAAAAAAMBgCIQAAAAAAAAMhreMldH2P3S8p/N13LH9ns5nC2vWrNHo0aOVk5NTYpv4+Hht3LhRBw8evGd1AQAAAABQ2dl8hdCbb74pPz8/OTk5KSwsTHv27Cm1/fz58/XQQw/J2dlZvr6+GjNmjK5du3aPqsX97sKFC4qMjJSbm5s8PDw0bNgwXb58udQ+nTp1kslksvo8//zzt53r/fffV0BAgJycnBQUFKSPP/641PYpKSlF5jGZTDp79uwdXSMAAAAAALdj0xVC7777ruLi4rR06VKFhYVp/vz56tq1q9LT01WrVq0i7ZOSkjRhwgStWrVKbdu21bfffqvo6GiZTCbNmzfPBleA+01kZKTOnDmjrVu36saNGxoyZIieffZZJSUlldpvxIgRevnlly3fXVxcSm3/xRdfaODAgUpMTNSf/vQnJSUlqXfv3tq/f78eeeSRUvump6fLzc3N8r24/yzAmDJfDiqfgR5wu32be2jxi/8ot7EuZq0tl3Em//2DchkHAAAAqKhsukJo3rx5GjFihIYMGaKmTZtq6dKlcnFx0apVq4pt/8UXX6hdu3YaNGiQ/Pz89Nhjj2ngwIG3XVVkBGazWYmJifL395ezs7OaNWumDz64+Q80t1aeJCcnKzQ0VC4uLmrbtq3S09Mt/Q8dOqRHH31U1atXl5ubm1q2bKm0tDTL+dTUVHXo0MGyMmvUqFG6cuWK5byfn59mzZqlqKgoubq6qn79+tq0aZPOnTunXr16ydXVVcHBwVZj3rJx40Y1btxYTk5O6tq1q06dOlXqta5YsUKBgYFycnJSQECA3nrrrTL9RkePHtWWLVu0YsUKhYWFqX379lq0aJE2bNigH374odS+Li4u8vb2tnx+GdgUZ8GCBerWrZvGjRunwMBAzZw5Uy1atNDixYtvW2etWrWs5rKzs/lCPgAAAABAJWOzf9K8fv269u3bp4iIiP8WY2eniIgI7dq1q9g+bdu21b59+ywB0PHjx/Xxxx/r8ccfL3GevLw8Xbp0yepTGSUmJmrdunVaunSpvvnmG40ZM0ZPP/20tm//715EkydP1ty5c5WWliYHBwcNHTrUci4yMlJ169bV3r17tW/fPk2YMEFVqlSRJGVkZKhbt27q27evDh8+rHfffVepqamKiYmxquGNN95Qu3btdODAAfXo0UPPPPOMoqKi9PTTT2v//v1q2LChoqKiVFhYaOmTm5ur2bNna926ddq5c6dycnI0YMCAEq9z/fr1mjZtmmbPnq2jR48qISFBU6dO1dq1t18VsGvXLnl4eCg0NNRyLCIiQnZ2dtq9e3epfdevX68aNWrokUce0cSJE5Wbm2t13s/PT/Hx8VZz/fLelqSuXbuWeG//UkhIiGrXrq0uXbpo586dt20PAAAAAMCdstkjY+fPn1dBQYG8vLysjnt5eek///lPsX0GDRqk8+fPq3379iosLFR+fr6ef/55TZo0qcR5EhMTNWPGjHKtvaLJy8tTQkKCtm3bpvDwcElSgwYNlJqaqmXLlunZZ5+VJM2ePVsdO97cHHvChAnq0aOHrl27JicnJ2VmZmrcuHEKCAiQJDVu3NgyfmJioiIjIzV69GjLuYULF6pjx45asmSJnJycJEmPP/64nnvuOUnStGnTtGTJErVq1Ur9+vWTJI0fP17h4eHKysqSt7e3JOnGjRtavHixwsLCJElr165VYGCg9uzZo9atWxe51unTp2vu3Lnq06ePJMnf319HjhzRsmXLNHjw4FJ/p7NnzxZ5/MrBwUGenp6l7tMzaNAg1a9fXz4+Pjp8+LDGjx+v9PR0ffjhh5Y2DRs2VI0aNazmKu7eLm2e2rVra+nSpQoNDVVeXp5WrFihTp06affu3WrRokWp1wYAAAAAwJ24r94ylpKSooSEBL311lsKCwvTsWPHFBsbq5kzZ2rq1KnF9pk4caLi4uIs3y9duiRfX997VfI9cezYMeXm5qpLly5Wx69fv67mzZtbvgcHB1v+rl27tiQpOztb9erVU1xcnIYPH66//e1vioiIUL9+/dSwYUNJNx8nO3z4sNavX2/pX1hYKLPZrBMnTigwMLDI+LfCkKCgoCLHsrOzLYGQg4ODWrVqZWkTEBAgDw8PHT16tEggdOXKFWVkZGjYsGEaMWKE5Xh+fr7c3d3L/HvdqVuBmnTzemrXrq3OnTsrIyPD8hslJyff9TwPPfSQHnroIcv3tm3bKiMjQ2+88Yb+9re/3fX4AAAAAADcYrNAqEaNGrK3t1dWVpbV8V+uHvm1qVOn6plnntHw4cMl3fyH8ytXrujZZ5/V5MmTi91rxdHRUY6OjuV/ARXIrbdkbd68WXXq1LE65+joqIyMDEmyPAImSSaTSdLNvYekm693HzRokDZv3qxPPvlE06dP14YNG/TEE0/o8uXLeu655zRq1Kgic9erV8/yd3Hjlzbnb73O5cuXW1YU3WJvb3/b/t7e3srOzrY6lp+frwsXLpR4zxXn1tzHjh2zBELFzXUn93ZJWrdurdTU1DvqAwAAAADA7dhsD6GqVauqZcuWVisrzGazkpOTLY89/Vpubm6R0OdWEPDLfWmMpmnTpnJ0dFRmZqYaNWpk9bmT1VBNmjTRmDFj9Omnn6pPnz5avXq1JKlFixY6cuRIkbEbNWqkqlWr3lXt+fn5VhtNp6enKycnx7Lq6Je8vLzk4+Oj48ePF6nD39//tnOFh4crJydH+/btsxz77LPPZDabiwRMpTl48KCk/66yKmmuX68a2rp1a4n3dmlzlTYPAAAAAAC/hU0fGYuLi9PgwYMVGhqq1q1ba/78+bpy5YqGDBkiSYqKilKdOnWUmJgoSerZs6fmzZun5s2bWx4Zmzp1qnr27FmmFSKVVfXq1TV27FiNGTNGZrNZ7du318WLF7Vz5065ubmpfv36pfa/evWqxo0bpyeffFL+/v76/vvvtXfvXvXt21fSzb1/2rRpo5iYGA0fPlzVqlXTkSNHtHXr1jK9Nas0VapU0ciRI7Vw4UI5ODgoJiZGbdq0KXb/IEmaMWOGRo0aJXd3d3Xr1k15eXlKS0vTTz/9ZPVoYHECAwPVrVs3jRgxQkuXLtWNGzcUExOjAQMGyMfHR5J0+vRpde7cWevWrVPr1q2VkZGhpKQkPf7443rwwQd1+PBhjRkzRn/4wx+sHpHr3LmznnjiCctG27GxserYsaPmzp2rHj16aMOGDUpLS9Pbb79t6TNx4kSdPn1a69atkyTNnz9f/v7+evjhh3Xt2jWtWLFCn332mT799NO7+o0BAAAAAPg1mwZC/fv317lz5zRt2jSdPXtWISEh2rJli2WvmczMTKsVQVOmTJHJZNKUKVN0+vRp1axZUz179tTs2bN/91o77th++0Y2NHPmTNWsWVOJiYk6fvy4PDw81KJFC02aNOm2j2jZ29vrxx9/VFRUlLKyslSjRg316dPHshl3cHCwtm/frsmTJ6tDhw4qLCxUw4YN1b9//7uu28XFRePHj9egQYN0+vRpdejQQStXriyx/fDhw+Xi4qLXXntN48aNU7Vq1RQUFGTZ8Pp21q9fr5iYGHXu3Fl2dnbq27evFi5caDl/48YNpaenW94iVrVqVW3bts0SVvr6+qpv376aMmWK1bgZGRk6f/685Xvbtm2VlJSkKVOmaNKkSWrcuLE2btyoRx55xNLmzJkzyszMtHy/fv26XnzxRZ0+fVouLi4KDg7Wtm3b9Oijj5bp2gAAAAAAKCubbyodExNT5PXlt6SkpFh9d3Bw0PTp0zV9+vR7UNn9xWQyKTY2VrGxscWe//UjdSEhIVbH3nnnnVLHb9WqVakrVU6ePHnbOf38/KyORUdHKzo6WpIsbw37tfj4eKvXuUs33/o1aNCgUustiaenp5KSkko8/+safX19tX377cPA4q6/X79+ljesFWfNmjVW31966SW99NJLt50LAAAAAIC7ZfNACKgMjpw6f/tGZdTUt8btG5VBu0XtymWcnSN3lss4AAAAAICKw2abSgPlLSEhQa6ursV+unfvbuvyAAAAAACoMFghhErj+eef11NPPVXsOWdn53tcDQAAAAAAFReBECoNT09PeXp62roMAAAAAAAqPAIhAKXa/oeO5TJORX9THwAAAAAYCXsIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAjBZtasWSMPD49S28THxyskJOSe1AMAAAAAgFHwlrEyWvziP+7pfDFze97T+YziwoULGjlypP7xj3/Izs5Offv21YIFC+Tq6lps+5MnT8rf37/Yc++995769etX4lz/+uf/atHcOTr9/SnV92uguIlT9Yc/dimx/b49X2pe4svKPJGh3Nxc1a9fX88995zGjBlzZxcJAAAAAMBtsEIIhhIZGalvvvlGW7du1T//+U/t2LFDzz77bIntfX19debMGavPjBkz5Orqqu7du5fY70DaHo0b+Zz69I/UBx9/pj927a6RIwbru/SjJfZxcXHRoOjh2rFjh44ePaopU6ZoypQpevvtt+/qmgEAAAAA+DUCoUrCbDYrMTFR/v7+cnZ2VrNmzfTBBx9IklJSUmQymZScnKzQ0FC5uLiobdu2Sk9Pt/Q/dOiQHn30UVWvXl1ubm5q2bKl0tLSLOdTU1PVoUMHOTs7y9fXV6NGjdKVK1cs5/38/DRr1ixFRUXJ1dVV9evX16ZNm3Tu3Dn16tVLrq6uCg4Othrzlo0bN6px48ZycnJS165dderUqVKvdcWKFQoMDJSTk5MCAgL01ltvlek3Onr0qLZs2aIVK1YoLCxM7du316JFi7Rhwwb98MMPxfaxt7eXt7e31eejjz7SU089VeKqIkn6+6q31b7jHzX0+Rg1bNxEo8ZOVNNHgpW0ZmWJfQIfCVaPXn308MMPy8/PT08//bS6du2qzz//vEzXBwAAAABAWREIVRKJiYlat26dli5dqm+++UZjxozR008/re3bt1vaTJ48WXPnzlVaWpocHBw0dOhQy7nIyEjVrVtXe/fu1b59+zRhwgRVqVJFkpSRkaFu3bqpb9++Onz4sN59912lpqYqJibGqoY33nhD7dq104EDB9SjRw8988wzioqK0tNPP639+/erYcOGioqKUmFhoaVPbm6uZs+erXXr1mnnzp3KycnRgAEDSrzO9evXa9q0aZo9e7aOHj2qhIQETZ06VWvXrr3tb7Rr1y55eHgoNDTUciwiIkJ2dnbavXv37X9kSfv27dPBgwc1bNgwq+Nd2rbQm/NetXw/uD9Nbdr/wapNuz88qoP7iwZiJTlw4IC++OILdezYscx9AAAAAAAoC/YQqgTy8vKUkJCgbdu2KTw8XJLUoEEDpaamatmyZZZHombPnm0JFyZMmKAePXro2rVrcnJyUmZmpsaNG6eAgABJUuPGjS3jJyYmKjIyUqNHj7acW7hwoTp27KglS5bIyclJkvT444/rueeekyRNmzZNS5YsUatWrSz77IwfP17h4eHKysqSt7e3JOnGjRtavHixwsLCJElr165VYGCg9uzZo9atWxe51unTp2vu3Lnq06ePJMnf319HjhzRsmXLNHjw4FJ/p7Nnz6pWrVpWxxwcHOTp6amzZ8+W5afWypUrFRgYqLZt21od963nJw9PT8v38+ey9WBN67kerFlTP57Lvu0cdevW1blz55Sfn6/4+HgNHz68TLUBAAAAAFBWBEKVwLFjx5Sbm6suXaw3LL5+/bqaN29u+R4cHGz5u3bt2pKk7Oxs1atXT3FxcRo+fLj+9re/KSIiQv369VPDhg0l3Xyc7PDhw1q/fr2lf2Fhocxms06cOKHAwMAi43t5eUmSgoKCihzLzs62BEIODg5q1aqVpU1AQIA8PDx09OjRIoHQlStXlJGRoWHDhmnEiBGW4/n5+XJ3dy/z7/VbXb16VUlJSZo6dWqRc6s2fFhu83z++ee6fPmyvvzyS02YMEGNGjXSwIEDy218AAAAAAAIhCqBy5cvS5I2b96sOnXqWJ1zdHRURkaGJFkeAZMkk8kk6ebeQ9LN17sPGjRImzdv1ieffKLp06drw4YNeuKJJ3T58mU999xzGjVqVJG569WrZ/m7uPFLm/O3Xufy5cstK4pusbe3v21/b29vZWdbr9DJz8/XhQsXLAFVaT744APl5uYqKirqtm1r1KxVZDXQj+fOFVk1VJxbbzULCgpSVlaW4uPjCYQAAAAAAOWKQKgSaNq0qRwdHZWZmVnsfjO3AqHbadKkiZo0aaIxY8Zo4MCBWr16tZ544gm1aNFCR44cUaNGjcq7dOXn5ystLc2yGig9PV05OTmWVUe/5OXlJR8fHx0/flyRkZF3PFd4eLhycnK0b98+tWzZUpL02WefyWw2FwmYirNy5Ur9+c9/Vs2aNW/bNqRFqL7c+bmihj9vObYrdbtCWoSW0qsos9msvLy8O+oDAAAAAMDtEAhVAtWrV9fYsWM1ZswYmc1mtW/fXhcvXtTOnTvl5uam+vXrl9r/6tWrGjdunJ588kn5+/vr+++/1969e9W3b19JN/f+adOmjWJiYjR8+HBVq1ZNR44c0datW7V48eK7qr1KlSoaOXKkFi5cKAcHB8XExKhNmzbF7h8kSTNmzNCoUaPk7u6ubt26KS8vT2lpafrpp58UFxdX6lyBgYHq1q2bRowYoaVLl+rGjRuKiYnRgAED5OPjI0k6ffq0OnfurHXr1lnVcOzYMe3YsUMff/xxsWMPHdBHnbs9rsjom/v9PD30WUU/1Utr3n5Lf/hjF32y6SN9ffig4ufMtfR5Y85MZZ89q8T5b0qSktauVG2fuqrS4ea8O3bs0Ouvv17syiwAAAAAAO4GgVAZxcztaesSSjVz5kzVrFlTiYmJOn78uDw8PNSiRQtNmjTpto9o2dvb68cff1RUVJSysrJUo0YN9enTRzNmzJB0c2+g7du3a/LkyerQoYMKCwvVsGFD9e/f/67rdnFx0fjx4zVo0CCdPn1aHTp00MqVJb+affjw4XJxcdFrr72mcePGqVq1agoKCrJseH0769evV0xMjDp37iw7Ozv17dtXCxcutJy/ceOG0tPTlZuba9Vv1apVqlu3rh577LFixz2VeVI5Fy5YvjcPba1XFy7VwtcTNf/V2arv10CLlq9V44f+u/LpXHaWzvzwveV7odms+a/M0ksjM+Xg4KCGDRvqlVdesWzUDQAAAABAeSEQqiRMJpNiY2MVGxtb7PlfvupdkkJCQqyOvfPOO6WO36pVK3366aclnj958uRt5/Tz87M6Fh0drejoaEmyvDXs1+Lj4xUfH291bNCgQRo0aFCp9ZbE09NTSUlJJZ7/dY23JCQkKCEhocR+W7/YX+RY1z/1Utc/9SqxT8I869VVkUNGKHLICDX1rVFiHwAAAAAAyoOdrQsAAAAAAADAvUUghEojISFBrq6uxX66d+9u6/IAAAAAAKgweGQMlcbzzz+vp556qthzzs7O97gaAAAAAAAqLgIhVBqenp7y9PS0dRkAAAAAAFR4PDIGAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEKwmTVr1sjDw6PUNvHx8QoJCbkn9QAAAAAAYBS8dr6MZj/95D2db/LfP7in8xnFhQsXNHLkSP3jH/+QnZ2d+vbtqwULFsjV1bXEPhkZGRo7dqxSU1OVl5enbt26adGiRfLy8rK0ycn5SQnTJipl279kZ2enLt3/pAnxs1WtWsnjvrd+nT7+3//Rka8P68rly9r11TG5ubuX6/UCAAAAAFAcVgjBUCIjI/XNN99o69at+uc//6kdO3bo2WefLbH9lStX9Nhjj8lkMumzzz7Tzp07df36dfXs2VNms9nSbvyo53Xs2/9oxfoP9Oaq9UrbvUvxE14stZZrV3PVruMfNeKF0eV1eQAAAAAAlAmBUCVhNpuVmJgof39/OTs7q1mzZvrgg5urjFJSUmQymZScnKzQ0FC5uLiobdu2Sk9Pt/Q/dOiQHn30UVWvXl1ubm5q2bKl0tLSLOdTU1PVoUMHOTs7y9fXV6NGjdKVK1cs5/38/DRr1ixFRUXJ1dVV9evX16ZNm3Tu3Dn16tVLrq6uCg4Othrzlo0bN6px48ZycnJS165dderUqVKvdcWKFQoMDJSTk5MCAgL01ltvlek3Onr0qLZs2aIVK1YoLCxM7du316JFi7Rhwwb98MMPxfbZuXOnTp48qTVr1igoKEhBQUFau3at0tLS9Nlnn1nGTU35TC+/Ml/BzVuqZes2mvRyoj7Z9JGyz54tsZ6o4c9rxAuxatYitEz1AwAAAABQXgiEKonExEStW7dOS5cu1TfffKMxY8bo6aef1vbt2y1tJk+erLlz5yotLU0ODg4aOnSo5VxkZKTq1q2rvXv3at++fZowYYKqVKki6eYjU926dVPfvn11+PBhvfvuu0pNTVVMTIxVDW+88YbatWunAwcOqEePHnrmmWcUFRWlp59+Wvv371fDhg0VFRWlwsJCS5/c3FzNnj1b69at086dO5WTk6MBAwaUeJ3r16/XtGnTNHv2bB09elQJCQmaOnWq1q5de9vfaNeuXfLw8FBo6H8DmIiICNnZ2Wn37t3F9snLy5PJZJKjo6PlmJOTk+zs7JSammoZ183NXY80C7G0CW/fUXZ2djp8cN9t6wIAAAAA4F5jD6FKIC8vTwkJCdq2bZvCw8MlSQ0aNFBqaqqWLVtmeSRq9uzZ6tixoyRpwoQJ6tGjh65duyYnJydlZmZq3LhxCggIkCQ1btzYMn5iYqIiIyM1evRoy7mFCxeqY8eOWrJkiZycnCRJjz/+uJ577jlJ0rRp07RkyRK1atVK/fr1kySNHz9e4eHhysrKkre3tyTpxo0bWrx4scLCwiRJa9euVWBgoPbs2aPWrVsXudbp06dr7ty56tOnjyTJ399fR44c0bJlyzR48OBSf6ezZ8+qVq1aVsccHBzk6empsyWs5GnTpo2qVaum8ePHKyEhQYWFhZowYYIKCgp05swZy7ieNWoUGdfd4wGdP5ddak0AAAAAANgCK4QqgWPHjik3N1ddunSRq6ur5bNu3TplZGRY2gUHB1v+rl27tiQpO/tmYBEXF6fhw4crIiJCc+bMsep36NAhrVmzxmrsrl27ymw268SJE8WOf2vD5aCgoCLHbs0p3QxOWrVqZfkeEBAgDw8PHT16tMh1XrlyRRkZGRo2bJhVLbNmzbKqtzzVrFlT77//vv7xj3/I1dVV7u7uysnJUYsWLWRnx398AAAAAAD3J1YIVQKXL1+WJG3evFl16tSxOufo6GgJS249AiZJJpNJkiwbI8fHx2vQoEHavHmzPvnkE02fPl0bNmzQE088ocuXL+u5557TqFGjisxdr149y9/FjV/anL/1OpcvX25ZUXSLvb39bft7e3tbhVGSlJ+frwsXLlhWLBXnscceU0ZGhs6fPy8HBwd5eHjI29tbDRo0sIx74fz5IuNezPlJNWrWKm5IAAAAAABsikCoEmjatKkcHR2VmZlpeSTsl8q6eqZJkyZq0qSJxowZo4EDB2r16tV64okn1KJFCx05ckSNGjUq79KVn5+vtLQ0y+Nh6enpysnJUWBgYJG2Xl5e8vHx0fHjxxUZGXnHc4WHhysnJ0f79u1Ty5YtJUmfffaZzGZzkYCpODX+/2Nhn332mbKzs/XnP//ZMu6lSxf1zeFDeji4mSRp9xefy2w2Kzik5R3XmffDN2Vuez3frPycbP3wZqwcrpyxPvmA2x3P/Xta/OI/ymWci1m33y+qLCb//YNyGQcAAAAA7kcEQpVA9erVNXbsWI0ZM0Zms1nt27fXxYsXtXPnTrm5ual+/fql9r969arGjRunJ598Uv7+/vr++++1d+9e9e3bV9LNvX/atGmjmJgYDR8+XNWqVdORI0e0detWLV68+K5qr1KlikaOHKmFCxfKwcFBMTExatOmTbH7B0nSjBkzNGrUKLm7u6tbt27Ky8tTWlqafvrpJ8XFxZU6V2BgoLp166YRI0Zo6dKlunHjhmJiYjRgwAD5+PhIkk6fPq3OnTtr3bp1lhpWr16twMBA1axZU7t27VJsbKzGjBmjhx56yDJu+05/1PQJYzQt4XXl37ih2VMnqPufn1Ct/7/yKOvsGQ0b2EcJb7yp4JAWkqRz2Vk6fy5bmSePS5K++88Rubi66gFfR3k+4H5XvysAAAAAAKUhECqjir6aYObMmapZs6YSExN1/PhxeXh4qEWLFpo0adJtH9Gyt7fXjz/+qKioKGVlZalGjRrq06ePZsyYIenm3kDbt2/X5MmT1aFDBxUWFqphw4bq37//Xdft4uKi8ePHa9CgQTp9+rQ6dOiglStXlth++PDhcnFx0WuvvaZx48apWrVqCgoKsmx4fTvr169XTEyMOnfuLDs7O/Xt21cLFy60nL9x44bS09OVm5trOZaenq6JEyfqwoUL8vPz0+TJkzVmzBircV9ZuFSzp07QsIF9ZGdnpy7d/6SJMxIs5/Nv3NCJjGO6dvWq5dh7f1+rt+a/Zvke1e/miqO3581SVP/eZboeAAAAAAB+CwKhSsJkMik2NlaxsbHFnv/lq94lKSQkxOrYO++8U+r4rVq10qefflri+ZMnT952Tj8/P6tj0dHRio6OliTLW8N+LT4+XvHx8VbHBg0apEGDBpVab0k8PT2VlJRU4vlf1yhJc+bM0Zw5c0od18PjAb22aFmJ5+v41tM3meesjr0Q95JeiHupSNuG9lmlzgUAAAAAwN3iNUkAAAAAAAAGQyCESiMhIcHqdfS//HTv3t3W5QEAAAAAUGHwyBgqjeeff15PPfVUseecnZ3vcTUAAAAAAFRcBEKoNDw9PeXp6WnrMgAAAAAAqPB4ZKwYv95UGKio/nurlv4mOQAAAAAAfolA6BeqVKkiSVavHAcqshvmQsmcL7sb3LMAAAAAgLLjkbFfsLe3l4eHh7KzsyVJLi4uMplMNq4K9wNz/o1yGyuvsGyrfcyF0o8Xc2V37ohM1y+X2/wAAAAAgMqPQOhXvL29JckSCgFlkf1T+QUyhaZLZW2pwqsXVT39Q5nEY44AAAAAgLIjEPoVk8mk2rVrq1atWrpxo/xWfaByG/v+xnIba1G1VWVqZyoskN3VH2UqLCi3uQEAAAAAxkAgVAJ7e3vZ29vbugzcJ87+fL3cxnLQmXIbCwAAAACA4rCpNAAAAAAAgMGwQugOtBy3rlzG2fdaVLmM025Ru3IZZ+fIneUyDgAAAAAAuD+wQggAAAAAAMBgCIQAAAAAAAAMhkAIAAAAAADAYAiEAAAAAAAADIZACAAAAAAAwGAIhAAAAAAAAAyGQAgAAAAAAMBgCIQAAAAAAAAMhkAIAAAAAADAYAiEAAAAAAAADIZACAAAAAAAwGAIhAAAAAAAAAyGQAgAAAAAAMBgCIQAAAAAAAAMhkAIAAAAAADAYAiEAAAAAAAADIZACAAAAAAAwGAIhAAAAAAAAAyGQAgAAAAAAMBgCIQAAAAAAAAMhkAIAAAAAADAYAiEAAAAAAAADIZACAAAAAAAwGAIhAAAAAAAAAyGQAgAAAAAAMBgbB4Ivfnmm/Lz85OTk5PCwsK0Z8+eUtvn5OTohRdeUO3ateXo6KgmTZro448/vkfVAgAAAAAA3P8cbDn5u+++q7i4OC1dulRhYWGaP3++unbtqvT0dNWqVatI++vXr6tLly6qVauWPvjgA9WpU0f/93//Jw8Pj3tfPAAAAAAAwH3KpoHQvHnzNGLECA0ZMkSStHTpUm3evFmrVq3ShAkTirRftWqVLly4oC+++EJVqlSRJPn5+d3LkgEAAAAAAO57Nntk7Pr169q3b58iIiL+W4ydnSIiIrRr165i+2zatEnh4eF64YUX5OXlpUceeUQJCQkqKCgocZ68vDxdunTJ6gMAAAAAAGBkNguEzp8/r4KCAnl5eVkd9/Ly0tmzZ4vtc/z4cX3wwQcqKCjQxx9/rKlTp2ru3LmaNWtWifMkJibK3d3d8vH19S3X6wAAAAAAALjf2HxT6TthNptVq1Ytvf3222rZsqX69++vyZMna+nSpSX2mThxoi5evGj5nDp16h5WDAAAAAAAUPHYbA+hGjVqyN7eXllZWVbHs7Ky5O3tXWyf2rVrq0qVKrK3t7ccCwwM1NmzZ3X9+nVVrVq1SB9HR0c5OjqWb/EAAAAAAAD3MZutEKpatapatmyp5ORkyzGz2azk5GSFh4cX26ddu3Y6duyYzGaz5di3336r2rVrFxsGAQAAAAAAoCibPjIWFxen5cuXa+3atTp69Kj+8pe/6MqVK5a3jkVFRWnixImW9n/5y1904cIFxcbG6ttvv9XmzZuVkJCgF154wVaXAAAAAAAAcN+x6Wvn+/fvr3PnzmnatGk6e/asQkJCtGXLFstG05mZmbKz+29m5evrq3/9618aM2aMgoODVadOHcXGxmr8+PG2ugQAAAAAAID7jk0DIUmKiYlRTExMsedSUlKKHAsPD9eXX375O1cFAAAAAABQed1XbxkDAAAAAADA3SMQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYB1sXYESZLweVz0APuJXLMNv/0LFcxvmq1dhyGedi1tpyGWfy3z8ol3EAAAAAAKhsWCEEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDC/KRD64x//qJycnCLHL126pD/+8Y93WxMAAAAAAAB+R78pEEpJSdH169eLHL927Zo+//zzuy4KAAAAAAAAvx+HO2l8+PBhy99HjhzR2bNnLd8LCgq0ZcsW1alTp/yqAwAAAAAAQLm7o0AoJCREJpNJJpOp2EfDnJ2dtWjRonIrDgAAAAAAAOXvjgKhEydOqLCwUA0aNNCePXtUs2ZNy7mqVauqVq1asre3L/ciAQAAAAAAUH7uKBCqX7++JMlsNv8uxQAAAAAAAOD3V+ZAaNOmTerevbuqVKmiTZs2ldr2z3/+810XBgAAAAAAgN9HmQOh3r176+zZs6pVq5Z69+5dYjuTyaSCgoLyqA0AAAAAAAC/gzIHQr98TIxHxgAAAAAAAO5fd7SH0C8lJycrOTlZ2dnZVgGRyWTSypUry6U4AAAAAAAAlL/fFAjNmDFDL7/8skJDQ1W7dm2ZTKbyrgsAAAAAAAC/k98UCC1dulRr1qzRM888U971AAAAAAAA4Hdm91s6Xb9+XW3bti3vWgAAAAAAAHAP/KZAaPjw4UpKSirvWgAAAAAAAHAPlPmRsbi4OMvfZrNZb7/9trZt26bg4GBVqVLFqu28efPKr0IAAAAAAACUqzIHQgcOHLD6HhISIkn6+uuvrY6zwTQAAAAAAEDFVuZA6N///vfvWQcAAAAAAADukd+0hxAAAAAAAADuXwRCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGUyECoTfffFN+fn5ycnJSWFiY9uzZU6Z+GzZskMlkUu/evX/fAgEAAAAAACoRmwdC7777ruLi4jR9+nTt379fzZo1U9euXZWdnV1qv5MnT2rs2LHq0KHDPaoUAAAAAACgcrB5IDRv3jyNGDFCQ4YMUdOmTbV06VK5uLho1apVJfYpKChQZGSkZsyYoQYNGtzDagEAAAAAAO5/Ng2Erl+/rn379ikiIsJyzM7OThEREdq1a1eJ/V5++WXVqlVLw4YNu+0ceXl5unTpktUHAAAAAADAyGwaCJ0/f14FBQXy8vKyOu7l5aWzZ88W2yc1NVUrV67U8uXLyzRHYmKi3N3dLR9fX9+7rhsAAAAAAOB+ZvNHxu7Ezz//rGeeeUbLly9XjRo1ytRn4sSJunjxouVz6tSp37lKAAAAAACAis3BlpPXqFFD9vb2ysrKsjqelZUlb2/vIu0zMjJ08uRJ9ezZ03LMbDZLkhwcHJSenq6GDRta9XF0dJSjo+PvUD0AAAAAAMD9yaYrhKpWraqWLVsqOTnZcsxsNis5OVnh4eFF2gcEBOirr77SwYMHLZ8///nPevTRR3Xw4EEeBwMAAAAAACgDm64QkqS4uDgNHjxYoaGhat26tebPn68rV65oyJAhkqSoqCjVqVNHiYmJcnJy0iOPPGLV38PDQ5KKHAcAAAAAAEDxbB4I9e/fX+fOndO0adN09uxZhYSEaMuWLZaNpjMzM2Vnd19tdQQAAAAAAFCh2TwQkqSYmBjFxMQUey4lJaXUvmvWrCn/ggAAAAAAACoxlt4AAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYDIEQAAAAAACAwRAIAQAAAAAAGAyBEAAAAAAAgMEQCAEAAAAAABgMgRAAAAAAAIDBEAgBAAAAAAAYTIUIhN588035+fnJyclJYWFh2rNnT4ltly9frg4dOuiBBx7QAw88oIiIiFLbAwAAAAAAwJrNA6F3331XcXFxmj59uvbv369mzZqpa9euys7OLrZ9SkqKBg4cqH//+9/atWuXfH199dhjj+n06dP3uHIAAAAAAID7k80DoXnz5mnEiBEaMmSImjZtqqVLl8rFxUWrVq0qtv369ev117/+VSEhIQoICNCKFStkNpuVnJx8jysHAAAAAAC4P9k0ELp+/br27duniIgIyzE7OztFRERo165dZRojNzdXN27ckKenZ7Hn8/LydOnSJasPAAAAAACAkdk0EDp//rwKCgrk5eVlddzLy0tnz54t0xjjx4+Xj4+PVaj0S4mJiXJ3d7d8fH1977puAAAAAACA+5nNHxm7G3PmzNGGDRv00UcfycnJqdg2EydO1MWLFy2fU6dO3eMqAQAAAAAAKhYHW05eo0YN2dvbKysry+p4VlaWvL29S+37+uuva86cOdq2bZuCg4NLbOfo6ChHR8dyqRcAAAAAAKAysOkKoapVq6ply5ZWG0Lf2iA6PDy8xH6vvvqqZs6cqS1btig0NPRelAoAAAAAAFBp2HSFkCTFxcVp8ODBCg0NVevWrTV//nxduXJFQ4YMkSRFRUWpTp06SkxMlCS98sormjZtmpKSkuTn52fZa8jV1VWurq42uw4AAAAAAID7hc0Dof79++vcuXOaNm2azp49q5CQEG3ZssWy0XRmZqbs7P67kGnJkiW6fv26nnzySatxpk+frvj4+HtZOgAAAAAAwH3J5oGQJMXExCgmJqbYcykpKVbfT548+fsXBAAAAAAAUInd128ZAwAAAAAAwJ0jEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIMhEAIAAAAAADAYAiEAAAAAAACDIRACAAAAAAAwGAIhAAAAAAAAgyEQAgAAAAAAMBgCIQAAAAAAAIOpEIHQm2++KT8/Pzk5OSksLEx79uwptf3777+vgIAAOTk5KSgoSB9//PE9qhQAAAAAAOD+Z/NA6N1331VcXJymT5+u/fv3q1mzZuratauys7OLbf/FF19o4MCBGjZsmA4cOKDevXurd+/e+vrrr+9x5QAAAAAAAPcnmwdC8+bN04gRIzRkyBA1bdpUS5culYuLi1atWlVs+wULFqhbt24aN26cAgMDNXPmTLVo0UKLFy++x5UDAAAAAADcnxxsOfn169e1b98+TZw40XLMzs5OERER2rVrV7F9du3apbi4OKtjXbt21caNG4ttn5eXp7y8PMv3ixcvSpIuXbp0x/UW5F294z7F+blKQbmMk381v1zGuVI+w+hqXm65jHPtxo1yGee3/N/4tyqve0Pi/rgdI98f3Bu3x/1x9yrr/cG9UT64P0pn5PuDe6N09/LekLg/bsfI9wf3xu3d7/fHrfaFhYW3b1xoQ6dPny6UVPjFF19YHR83blxh69ati+1TpUqVwqSkJKtjb775ZmGtWrWKbT99+vRCSXz48OHDhw8fPnz48OHDhw8fPob4nDp16raZjE1XCN0LEydOtFpRZDabdeHCBT344IMymUw2rKxiuHTpknx9fXXq1Cm5ubnZuhxUMNwfKAn3BkrD/YHScH+gJNwbKA33B0rCvWGtsLBQP//8s3x8fG7b1qaBUI0aNWRvb6+srCyr41lZWfL29i62j7e39x21d3R0lKOjo9UxDw+P3150JeXm5sZ/eFAi7g+UhHsDpeH+QGm4P1AS7g2UhvsDJeHe+C93d/cytbPpptJVq1ZVy5YtlZycbDlmNpuVnJys8PDwYvuEh4dbtZekrVu3ltgeAAAAAAAA1mz+yFhcXJwGDx6s0NBQtW7dWvPnz9eVK1c0ZMgQSVJUVJTq1KmjxMRESVJsbKw6duyouXPnqkePHtqwYYPS0tL09ttv2/IyAAAAAAAA7hs2D4T69++vc+fOadq0aTp79qxCQkK0ZcsWeXl5SZIyMzNlZ/ffhUxt27ZVUlKSpkyZokmTJqlx48bauHGjHnnkEVtdwn3N0dFR06dPL/JYHSBxf6Bk3BsoDfcHSsP9gZJwb6A03B8oCffGb2cqLCzLu8gAAAAAAABQWdh0DyEAAAAAAADcewRCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIGRQO3bsUM+ePeXj4yOTyaSNGzfauiRUEImJiWrVqpWqV6+uWrVqqXfv3kpPT7d1WagglixZouDgYLm5ucnNzU3h4eH65JNPbF0WKqA5c+bIZDJp9OjRti4FFUB8fLxMJpPVJyAgwNZloQI5ffq0nn76aT344INydnZWUFCQ0tLSbF0WKgA/P78i//1hMpn0wgsv2Lo02FhBQYGmTp0qf39/OTs7q2HDhpo5c6Z4b1bZ2fy187CNK1euqFmzZho6dKj69Olj63JQgWzfvl0vvPCCWrVqpfz8fE2aNEmPPfaYjhw5omrVqtm6PNhY3bp1NWfOHDVu3FiFhYVau3atevXqpQMHDujhhx+2dXmoIPbu3atly5YpODjY1qWgAnn44Ye1bds2y3cHB/5nKG766aef1K5dOz366KP65JNPVLNmTX333Xd64IEHbF0aKoC9e/eqoKDA8v3rr79Wly5d1K9fPxtWhYrglVde0ZIlS7R27Vo9/PDDSktL05AhQ+Tu7q5Ro0bZurz7Av+f2KC6d++u7t2727oMVEBbtmyx+r5mzRrVqlVL+/bt0x/+8AcbVYWKomfPnlbfZ8+erSVLlujLL78kEIIk6fLly4qMjNTy5cs1a9YsW5eDCsTBwUHe3t62LgMV0CuvvCJfX1+tXr3acszf39+GFaEiqVmzptX3OXPmqGHDhurYsaONKkJF8cUXX6hXr17q0aOHpJuryd555x3t2bPHxpXdP3hkDECpLl68KEny9PS0cSWoaAoKCrRhwwZduXJF4eHhti4HFcQLL7ygHj16KCIiwtaloIL57rvv5OPjowYNGigyMlKZmZm2LgkVxKZNmxQaGqp+/fqpVq1aat68uZYvX27rslABXb9+XX//+981dOhQmUwmW5cDG2vbtq2Sk5P17bffSpIOHTqk1NRUFj7cAVYIASiR2WzW6NGj1a5dOz3yyCO2LgcVxFdffaXw8HBdu3ZNrq6u+uijj9S0aVNbl4UKYMOGDdq/f7/27t1r61JQwYSFhWnNmjV66KGHdObMGc2YMUMdOnTQ119/rerVq9u6PNjY8ePHtWTJEsXFxWnSpEnau3evRo0apapVq2rw4MG2Lg8VyMaNG5WTk6Po6Ghbl4IKYMKECbp06ZICAgJkb2+vgoICzZ49W5GRkbYu7b5BIASgRC+88IK+/vprpaam2roUVCAPPfSQDh48qIsXL+qDDz7Q4MGDtX37dkIhgzt16pRiY2O1detWOTk52bocVDC//Le1wcHBCgsLU/369fXee+9p2LBhNqwMFYHZbFZoaKgSEhIkSc2bN9fXX3+tpUuXEgjBysqVK9W9e3f5+PjYuhRUAO+9957Wr1+vpKQkPfzwwzp48KBGjx4tHx8f/rujjAiEABQrJiZG//znP7Vjxw7VrVvX1uWgAqlataoaNWokSWrZsqX27t2rBQsWaNmyZTauDLa0b98+ZWdnq0WLFpZjBQUF2rFjhxYvXqy8vDzZ29vbsEJUJB4eHmrSpImOHTtm61JQAdSuXbvIv1QIDAzU//zP/9ioIlRE//d//6dt27bpww8/tHUpqCDGjRunCRMmaMCAAZKkoKAg/d///Z8SExMJhMqIQAiAlcLCQo0cOVIfffSRUlJS2NQRt2U2m5WXl2frMmBjnTt31ldffWV1bMiQIQoICND48eMJg2Dl8uXLysjI0DPPPGPrUlABtGvXTunp6VbHvv32W9WvX99GFaEiWr16tWrVqmXZQBjIzc2VnZ31tsj29vYym802quj+QyBkUJcvX7b6t3InTpzQwYMH5enpqXr16tmwMtjaCy+8oKSkJP3v//6vqlevrrNnz0qS3N3d5ezsbOPqYGsTJ05U9+7dVa9ePf38889KSkpSSkqK/vWvf9m6NNhY9erVi+w1Vq1aNT344IPsQQaNHTtWPXv2VP369fXDDz9o+vTpsre318CBA21dGiqAMWPGqG3btkpISNBTTz2lPXv26O2339bbb79t69JQQZjNZq1evVqDBw+WgwP/CIubevbsqdmzZ6tevXp6+OGHdeDAAc2bN09Dhw61dWn3DVNhYWGhrYvAvZeSkqJHH320yPHBgwdrzZo1974gVBglvbFh9erVbOAHDRs2TMnJyTpz5ozc3d0VHBys8ePHq0uXLrYuDRVQp06dFBISovnz59u6FNjYgAEDtGPHDv3444+qWbOm2rdvr9mzZ6thw4a2Lg0VxD//+U9NnDhR3333nfz9/RUXF6cRI0bYuixUEJ9++qm6du2q9PR0NWnSxNbloIL4+eefNXXqVH300UfKzs6Wj4+PBg4cqGnTpqlq1aq2Lu++QCAEAAAAAABgMHa3bwIAAAAAAIDKhEAIAAAAAADAYAiEAAAAAAAADIZACAAAAAAAwGAIhAAAAAAAAAyGQAgAAAAAAMBgCIQAAAAAAAAMhkAIAACgAvHz89P8+fNtXQYAAKjkCIQAAABK0KlTJ40ePdrWZQAAAJQ7AiEAAAAAAACDIRACAAAoRnR0tLZv364FCxbIZDLJZDLp5MmT2r59u1q3bi1HR0fVrl1bEyZMUH5+vqVfp06dFBMTo5iYGLm7u6tGjRqaOnWqCgsLf1MdK1askIeHh5KTk8vr0gAAAAiEAAAAirNgwQKFh4drxIgROnPmjM6cOaMqVaro8ccfV6tWrXTo0CEtWbJEK1eu1KxZs6z6rl27Vg4ODtqzZ48WLFigefPmacWKFXdcw6uvvqoJEybo008/VefOncvr0gAAAORg6wIAAAAqInd3d1WtWlUuLi7y9vaWJE2ePFm+vr5avHixTCaTAgIC9MMPP2j8+PGaNm2a7Oxu/rs2X19fvfHGGzKZTHrooYf01Vdf6Y033tCIESPKPP/48eP1t7/9Tdu3b9fDDz/8u1wjAAAwLlYIAQAAlNHRo0cVHh4uk8lkOdauXTtdvnxZ33//veVYmzZtrNqEh4fru+++U0FBQZnmmTt3rpYvX67U1FTCIAAA8LsgEAIAAKhgOnTooIKCAr333nu2LgUAAFRSBEIAAAAlqFq1qtWqnsDAQO3atctqg+idO3eqevXqqlu3ruXY7t27rcb58ssv1bhxY9nb25dp3tatW+uTTz5RQkKCXn/99bu8CgAAgKIIhAAAAErg5+en3bt36+TJkzp//rz++te/6tSpUxo5cqT+85//6H//9381ffp0xcXFWfYPkqTMzEzFxcUpPT1d77zzjhYtWqTY2Ng7mrtt27b6+OOPNWPGDM2fP7+crwwAABgdm0oDAACUYOzYsRo8eLCaNm2qq1ev6sSJE/r44481btw4NWvWTJ6enho2bJimTJli1S8qKkpXr15V69atZW9vr9jYWD377LN3PH/79u21efNmPf7447K3t9fIkSPL69IAAIDBmQp/ueYZAAAAd6VTp04KCQlhVQ8AAKjQeGQMAAAAAADAYAiEAAAA7pHPP/9crq6uJX4AAADuFR4ZAwAAuEeuXr2q06dPl3i+UaNG97AaAABgZARCAAAAAAAABsMjYwAAAAAAAAZDIAQAAAAAAGAwBEIAAAAAAAAGQyAEAAAAAABgMARCAAAAAAAABkMgBAAAAAAAYDAEQgAAAAAAAAZDIAQAAAAAAGAw/w+WtTtgzVPyfwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1400x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "plt.figure(figsize=(14, 6))\n",
    "sns.barplot(x='top_k', y='hit', hue='retriever', data=hit_stat_df, errorbar=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7925564a-7d30-4914-baaf-4a00abb7686d",
   "metadata": {},
   "source": [
    "# 问答全流程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8587fd0d-81e4-44e0-9ee0-7c077081046e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T13:48:20.582089Z",
     "iopub.status.busy": "2024-08-08T13:48:20.581917Z",
     "iopub.status.idle": "2024-08-08T13:48:20.584631Z",
     "shell.execute_reply": "2024-08-08T13:48:20.584292Z",
     "shell.execute_reply.started": "2024-08-08T13:48:20.582076Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain.llms import Ollama\n",
    "\n",
    "llm = Ollama(\n",
    "    model='qwen2:7b-instruct',\n",
    "    base_url='http://localhost:11434'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d5d52868-4a9d-488e-8cce-55bb166e09ce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T13:48:20.840372Z",
     "iopub.status.busy": "2024-08-08T13:48:20.839621Z",
     "iopub.status.idle": "2024-08-08T13:48:29.365772Z",
     "shell.execute_reply": "2024-08-08T13:48:29.365291Z",
     "shell.execute_reply.started": "2024-08-08T13:48:20.840303Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'我是阿里云开发的一款超大规模语言模型，我叫通义千问。作为一个AI助手，我的目标是帮助用户获得准确、有用的信息，解决他们的问题和困惑。无论是学术问题、技术难题、日常建议还是娱乐话题，我都尽力提供高质量的回答或辅助。请随时向我提问，我会努力提供最好的帮助！'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.invoke('你是谁')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "df5850ea-3f9c-4834-9de1-755d600b05b6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T13:48:29.366707Z",
     "iopub.status.busy": "2024-08-08T13:48:29.366432Z",
     "iopub.status.idle": "2024-08-08T13:48:29.369757Z",
     "shell.execute_reply": "2024-08-08T13:48:29.369427Z",
     "shell.execute_reply.started": "2024-08-08T13:48:29.366693Z"
    }
   },
   "outputs": [],
   "source": [
    "def rag(retriever, query, n_chunks=4):\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个金融分析师，擅长根据所获取的信息片段，对问题进行分析和推理。\n",
    "你的任务是根据所获取的信息片段（<<<<context>>><<<</context>>>之间的内容）回答问题。\n",
    "回答保持简洁，不必重复问题，不要添加描述性解释和与答案无关的任何内容。\n",
    "已知信息：\n",
    "<<<<context>>>\n",
    "{{knowledge}}\n",
    "<<<</context>>>\n",
    "\n",
    "问题：{{query}}\n",
    "请回答：\n",
    "\"\"\".strip()\n",
    "\n",
    "    # chunks = retrieve(vector_db, query, k=n_chunks)\n",
    "    chunks = retriever.get_relevant_documents(query)[:n_chunks]\n",
    "    prompt = prompt_tmpl.replace('{{knowledge}}', '\\n\\n'.join([doc.page_content for doc in chunks])).replace('{{query}}', query)\n",
    "\n",
    "    return llm(prompt), chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7cf52672-6e47-458d-b050-51cb45178184",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T13:48:35.917955Z",
     "iopub.status.busy": "2024-08-08T13:48:35.917185Z",
     "iopub.status.idle": "2024-08-08T13:48:41.565895Z",
     "shell.execute_reply": "2024-08-08T13:48:41.565393Z",
     "shell.execute_reply.started": "2024-08-08T13:48:35.917885Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.10/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The method `BaseLLM.__call__` was deprecated in langchain-core 0.1.7 and will be removed in 0.3.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023年10月美国ISM制造业PMI指数较上个月大幅下降了2.3个百分点。\n"
     ]
    }
   ],
   "source": [
    "n_chunks = 3\n",
    "\n",
    "retriever = build_get_ensemble_retriver_fn(weights=[0.5, 0.5])(n_chunks)\n",
    "print(rag(retriever, '2023年10月美国ISM制造业PMI指数较上月有何变化？')[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0606dfb4-494b-4101-9bbe-b4e7105d6626",
   "metadata": {},
   "source": [
    "## 预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "beba9763-119d-4519-b06e-d3290d153f3c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T13:48:42.722277Z",
     "iopub.status.busy": "2024-08-08T13:48:42.722064Z",
     "iopub.status.idle": "2024-08-08T13:48:42.726977Z",
     "shell.execute_reply": "2024-08-08T13:48:42.726586Z",
     "shell.execute_reply.started": "2024-08-08T13:48:42.722264Z"
    }
   },
   "outputs": [],
   "source": [
    "prediction_df = qa_df[qa_df['dataset'] == 'test'][['uuid', 'question', 'qa_type', 'answer']].rename(columns={'answer': 'ref_answer'})\n",
    "\n",
    "def predict(prediction_df, retriever, n_chunks):\n",
    "    prediction_df = prediction_df.copy()\n",
    "    answer_dict = {}\n",
    "\n",
    "    for idx, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        uuid = row['uuid']\n",
    "        question = row['question']\n",
    "        answer, chunks = rag(retriever, question, n_chunks=n_chunks)\n",
    "        assert len(chunks) <= n_chunks\n",
    "        answer_dict[question] = {\n",
    "            'uuid': uuid,\n",
    "            'ref_answer': row['ref_answer'],\n",
    "            'gen_answer': answer,\n",
    "            'chunks': chunks\n",
    "        }\n",
    "    prediction_df.loc[:, 'gen_answer'] = prediction_df['question'].apply(lambda q: answer_dict[q]['gen_answer'])\n",
    "    prediction_df.loc[:, 'chunks'] = prediction_df['question'].apply(lambda q: answer_dict[q]['chunks'])\n",
    "\n",
    "    return prediction_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "369e5bd3-d2f4-4b81-b072-d53775df3d97",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T13:50:44.576362Z",
     "iopub.status.busy": "2024-08-08T13:50:44.576144Z",
     "iopub.status.idle": "2024-08-08T14:29:59.799089Z",
     "shell.execute_reply": "2024-08-08T14:29:59.798618Z",
     "shell.execute_reply.started": "2024-08-08T13:50:44.576347Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81751efde9a7444b813d15f1659d9839",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_chunks = 3\n",
    "retriver = build_get_ensemble_retriver_fn(weights=[0.5, 0.5])(n_chunks)\n",
    "pred_df = predict(prediction_df, retriever, n_chunks=n_chunks)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20639664-cc94-4057-b2bb-00d574b9d6c0",
   "metadata": {},
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a8184601-720c-4f0e-8b54-bdd84deb3051",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T14:48:41.056107Z",
     "iopub.status.busy": "2024-08-08T14:48:41.055937Z",
     "iopub.status.idle": "2024-08-08T14:48:41.071658Z",
     "shell.execute_reply": "2024-08-08T14:48:41.071154Z",
     "shell.execute_reply.started": "2024-08-08T14:48:41.056094Z"
    }
   },
   "outputs": [],
   "source": [
    "from langchain_openai import ChatOpenAI\n",
    "import time\n",
    "\n",
    "judge_llm = ChatOpenAI(\n",
    "    api_key=os.environ['LLM_API_KEY'],\n",
    "    base_url=os.environ['LLM_BASE_URL'],\n",
    "    model_name='qwen2-72b-instruct',\n",
    "    temperature=0\n",
    ")\n",
    "\n",
    "def evaluate(prediction_df):\n",
    "    \"\"\"\n",
    "    对预测结果进行打分\n",
    "    :param prediction_df: 预测结果，需要包含问题，参考答案，生成的答案，列名分别为question, ref_answer, gen_answer\n",
    "    :return 打分模型原始返回结果\n",
    "    \"\"\"\n",
    "    prompt_tmpl = \"\"\"\n",
    "你是一个经济学博士，现在我有一系列问题，有一个助手已经对这些问题进行了回答，你需要参照参考答案，评价这个助手的回答是否正确，仅回复“是”或“否”即可，不要带其他描述性内容或无关信息。\n",
    "问题：\n",
    "<question>\n",
    "{{question}}\n",
    "</question>\n",
    "\n",
    "参考答案：\n",
    "<ref_answer>\n",
    "{{ref_answer}}\n",
    "</ref_answer>\n",
    "\n",
    "助手回答：\n",
    "<gen_answer>\n",
    "{{gen_answer}}\n",
    "</gen_answer>\n",
    "请评价：\n",
    "    \"\"\"\n",
    "    results = []\n",
    "\n",
    "    for _, row in tqdm(prediction_df.iterrows(), total=len(prediction_df)):\n",
    "        question = row['question']\n",
    "        ref_answer = row['ref_answer']\n",
    "        gen_answer = row['gen_answer']\n",
    "\n",
    "        prompt = prompt_tmpl.replace('{{question}}', question).replace('{{ref_answer}}', str(ref_answer)).replace('{{gen_answer}}', gen_answer).strip()\n",
    "        result = judge_llm.invoke(prompt).content\n",
    "        results.append(result)\n",
    "\n",
    "        time.sleep(1)\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "ce596be7-0850-4f7e-9f00-1936333b2681",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T14:48:42.080568Z",
     "iopub.status.busy": "2024-08-08T14:48:42.080393Z",
     "iopub.status.idle": "2024-08-08T14:51:01.642457Z",
     "shell.execute_reply": "2024-08-08T14:51:01.642098Z",
     "shell.execute_reply.started": "2024-08-08T14:48:42.080555Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa424b6fe5e84fd48197977362fcd815",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pred_df['raw_score'] = evaluate(pred_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "08e44d7a-5567-407e-b8ba-2a9bf67fa159",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T14:56:22.236888Z",
     "iopub.status.busy": "2024-08-08T14:56:22.236713Z",
     "iopub.status.idle": "2024-08-08T14:56:22.239892Z",
     "shell.execute_reply": "2024-08-08T14:56:22.239557Z",
     "shell.execute_reply.started": "2024-08-08T14:56:22.236874Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['是', '否'], dtype=object)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df['raw_score'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "dd04ba60-3a62-4672-9f71-83f18da0b04e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T14:56:22.859830Z",
     "iopub.status.busy": "2024-08-08T14:56:22.859646Z",
     "iopub.status.idle": "2024-08-08T14:56:22.863279Z",
     "shell.execute_reply": "2024-08-08T14:56:22.862915Z",
     "shell.execute_reply.started": "2024-08-08T14:56:22.859815Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_172348/3896259812.py:2: FutureWarning: Downcasting behavior in `replace` is deprecated and will be removed in a future version. To retain the old behavior, explicitly call `result.infer_objects(copy=False)`. To opt-in to the future behavior, set `pd.set_option('future.no_silent_downcasting', True)`\n",
      "  pred_df.loc[:, 'score'] = pred_df['raw_score'].replace({'是': 1, '否': 0})\n"
     ]
    }
   ],
   "source": [
    "# prediction_df['score'] = (prediction_df['raw_score'] == '是').astype(int)\n",
    "pred_df.loc[:, 'score'] = pred_df['raw_score'].replace({'是': 1, '否': 0})\n",
    "_ = pred_df.pop('raw_score')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e7d776e2-1e79-4627-b035-f997db8a70dd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-08-08T14:56:23.492027Z",
     "iopub.status.busy": "2024-08-08T14:56:23.491826Z",
     "iopub.status.idle": "2024-08-08T14:56:23.495217Z",
     "shell.execute_reply": "2024-08-08T14:56:23.494878Z",
     "shell.execute_reply.started": "2024-08-08T14:56:23.492013Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.78"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_df['score'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b05ebb8f-0263-4e45-a8c2-a9eac9dee541",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_df.to_excel(os.path.join(expr_dir, 'prediction.xlsx'), index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
